Move storage classes to storage module

clcache.py

@@ -12,7 +12,6 @@
 import cProfile
 import codecs
 import concurrent.futures
-import contextlib
 import errno
 import hashlib
 import json
@@ -25,6 +24,7 @@
 import sys
 import threading
 from tempfile import TemporaryFile
+from storage import Storage
 
 VERSION = "4.1.0-dev"
 
@@ -199,18 +199,6 @@ def getManifest(self, manifestHash):
             return None
 
 
-@contextlib.contextmanager
-def allSectionsLocked(repository):
-    sections = list(repository.sections())
-    for section in sections:
-        section.lock.acquire()
-    try:
-        yield
-    finally:
-        for section in sections:
-            section.lock.release()
-
-
 class ManifestRepository(object):
     # Bump this counter whenever the current manifest file format changes.
     # E.g. changing the file format from {'oldkey': ...} to {'newkey': ...} requires
@@ -479,148 +467,6 @@ def _normalizedCommandLine(cmdline):
         return [arg for arg in cmdline
                 if not (arg[0] in "/-" and arg[1:].startswith(argsToStrip))]
 
-class CacheFileStrategy(object):
-    def __init__(self, cacheDirectory=None):
-        self.dir = cacheDirectory
-        if not self.dir:
-            try:
-                self.dir = os.environ["CLCACHE_DIR"]
-            except KeyError:
-                self.dir = os.path.join(os.path.expanduser("~"), "clcache")
-
-        manifestsRootDir = os.path.join(self.dir, "manifests")
-        ensureDirectoryExists(manifestsRootDir)
-        self.manifestRepository = ManifestRepository(manifestsRootDir)
-
-        compilerArtifactsRootDir = os.path.join(self.dir, "objects")
-        ensureDirectoryExists(compilerArtifactsRootDir)
-        self.compilerArtifactsRepository = CompilerArtifactsRepository(compilerArtifactsRootDir)
-
-        self.configuration = Configuration(os.path.join(self.dir, "config.txt"))
-        self.statistics = Statistics(os.path.join(self.dir, "stats.txt"))
-
-    def __str__(self):
-        return "Disk cache at {}".format(self.dir)
-
-    @property
-    @contextlib.contextmanager
-    def lock(self):
-        with allSectionsLocked(self.manifestRepository), \
-             allSectionsLocked(self.compilerArtifactsRepository), \
-             self.statistics.lock:
-            yield
-
-    def lockFor(self, key):
-        assert isinstance(self.compilerArtifactsRepository.section(key).lock, CacheLock)
-        return self.compilerArtifactsRepository.section(key).lock
-
-    def manifestLockFor(self, key):
-        return self.manifestRepository.section(key).lock
-
-    def getEntry(self, key):
-        return self.compilerArtifactsRepository.section(key).getEntry(key)
-
-    def setEntry(self, key, value):
-        self.compilerArtifactsRepository.section(key).setEntry(key, value)
-
-    def pathForObject(self, key):
-        return self.compilerArtifactsRepository.section(key).cachedObjectName(key)
-
-    def directoryForCache(self, key):
-        return self.compilerArtifactsRepository.section(key).cacheEntryDir(key)
-
-    def deserializeCacheEntry(self, key, objectData):
-        path = self.pathForObject(key)
-        ensureDirectoryExists(self.directoryForCache(key))
-        with open(path, 'wb') as f:
-            f.write(objectData)
-        return path
-
-    def hasEntry(self, cachekey):
-        return self.compilerArtifactsRepository.section(cachekey).hasEntry(cachekey)
-
-    def setManifest(self, manifestHash, manifest):
-        self.manifestRepository.section(manifestHash).setManifest(manifestHash, manifest)
-
-    def getManifest(self, manifestHash):
-        return self.manifestRepository.section(manifestHash).getManifest(manifestHash)
-
-    def clean(self, stats, maximumSize):
-        currentSize = stats.currentCacheSize()
-        if currentSize < maximumSize:
-            return
-
-        # Free at least 10% to avoid cleaning up too often which
-        # is a big performance hit with large caches.
-        effectiveMaximumSizeOverall = maximumSize * 0.9
-
-        # Split limit in manifests (10 %) and objects (90 %)
-        effectiveMaximumSizeManifests = effectiveMaximumSizeOverall * 0.1
-        effectiveMaximumSizeObjects = effectiveMaximumSizeOverall - effectiveMaximumSizeManifests
-
-        # Clean manifests
-        currentSizeManifests = self.manifestRepository.clean(effectiveMaximumSizeManifests)
-
-        # Clean artifacts
-        currentCompilerArtifactsCount, currentCompilerArtifactsSize = self.compilerArtifactsRepository.clean(
-            effectiveMaximumSizeObjects)
-
-        stats.setCacheSize(currentCompilerArtifactsSize + currentSizeManifests)
-        stats.setNumCacheEntries(currentCompilerArtifactsCount)
-
-
-class Cache(object):
-    def __init__(self, cacheDirectory=None):
-        if os.environ.get("CLCACHE_MEMCACHED"):
-            from storage import CacheFileWithMemcacheFallbackStrategy
-            self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"),
-                                                                  cacheDirectory=cacheDirectory)
-        else:
-            self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory)
-
-    def __str__(self):
-        return str(self.strategy)
-
-    @property
-    def lock(self):
-        return self.strategy.lock
-
-    @contextlib.contextmanager
-    def manifestLockFor(self, key):
-        with self.strategy.manifestLockFor(key):
-            yield
-
-    @property
-    def configuration(self):
-        return self.strategy.configuration
-
-    @property
-    def statistics(self):
-        return self.strategy.statistics
-
-    def clean(self, stats, maximumSize):
-        return self.strategy.clean(stats, maximumSize)
-
-    @contextlib.contextmanager
-    def lockFor(self, key):
-        with self.strategy.lockFor(key):
-            yield
-
-    def getEntry(self, key):
-        return self.strategy.getEntry(key)
-
-    def setEntry(self, key, value):
-        self.strategy.setEntry(key, value)
-
-    def hasEntry(self, cachekey):
-        return self.strategy.hasEntry(cachekey)
-
-    def setManifest(self, manifestHash, manifest):
-        self.strategy.setManifest(manifestHash, manifest)
-
-    def getManifest(self, manifestHash):
-        return self.strategy.getManifest(manifestHash)
-
 
 class PersistentJSONDict(object):
     def __init__(self, fileName):
@@ -654,31 +500,6 @@ def __eq__(self, other):
         return type(self) is type(other) and self.__dict__ == other.__dict__
 
 
-class Configuration(object):
-    _defaultValues = {"MaximumCacheSize": 1073741824} # 1 GiB
-
-    def __init__(self, configurationFile):
-        self._configurationFile = configurationFile
-        self._cfg = None
-
-    def __enter__(self):
-        self._cfg = PersistentJSONDict(self._configurationFile)
-        for setting, defaultValue in self._defaultValues.items():
-            if setting not in self._cfg:
-                self._cfg[setting] = defaultValue
-        return self
-
-    def __exit__(self, typ, value, traceback):
-        # Does not write to disc when unchanged
-        self._cfg.save()
-
-    def maximumCacheSize(self):
-        return self._cfg["MaximumCacheSize"]
-
-    def setMaximumCacheSize(self, size):
-        self._cfg["MaximumCacheSize"] = size
-
-
 class Statistics(object):
     CALLS_WITH_INVALID_ARGUMENT = "CallsWithInvalidArgument"
     CALLS_WITHOUT_SOURCE_FILE = "CallsWithoutSourceFile"
@@ -1505,7 +1326,7 @@ def main():
 """.strip().format(VERSION))
         return 0
 
-    cache = Cache()
+    cache = Storage()
 
     if len(sys.argv) == 2 and sys.argv[1] == "-s":
         with cache.lock:
@@ -1646,7 +1467,7 @@ def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles
 def processSingleSource(compiler, cmdLine, sourceFile, objectFile, environment):
     try:
         assert objectFile is not None
-        cache = Cache()
+        cache = Storage()
 
         if 'CLCACHE_NODIRECT' in os.environ:
             return processNoDirect(cache, objectFile, compiler, cmdLine, environment)