Reuse python diskcache instances (#58)

Each instance holds open file handles, and each time we instantiate a backend, we rack up open file handles until we run out of them. Test plan: Use this to successfully load the osmFISH data set in a python notebook.
spacetx · Sep 5, 2018 · dae184e · dae184e
1 parent f4087ce
commit dae184e
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/setup.py b/setup.py
@@ -7,7 +7,7 @@
 
 setuptools.setup(
     name="slicedimage",
-    version="0.0.4",
+    version="0.0.5",
     description="Library to access sliced imaging data",
     author="Tony Tung",
     author_email="[email protected]",

diff --git a/slicedimage/backends/_caching.py b/slicedimage/backends/_caching.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import io
+from threading import Lock
 
 from diskcache import Cache
 
@@ -11,10 +12,15 @@
 
 
 class CachingBackend(Backend):
+    _LOCK = Lock()
+    _CACHE = {}
+
     def __init__(self, cacheroot, authoritative_backend):
-        self._cacheroot = cacheroot
+        with CachingBackend._LOCK:
+            if cacheroot not in CachingBackend._CACHE:
+                CachingBackend._CACHE[cacheroot] = Cache(cacheroot, size=int(SIZE_LIMIT))
+            self._cache = CachingBackend._CACHE[cacheroot]
         self._authoritative_backend = authoritative_backend
-        self._cache = Cache(cacheroot, size_limit=int(SIZE_LIMIT))
 
     def read_contextmanager(self, name, checksum_sha256=None):
         if checksum_sha256 is not None: