Have memestra look for extra deprecation information in a shared dir

User or third-party package can populate <prefix>/share/memestra with yaml files
describing deprecated functions in order to speedup computation and / or provide
information not available from decorators.

Author: serge-sans-paille

docs/memestra-cache.rst

@@ -1,31 +1,79 @@
 Memestra-cache
 ==============
 
-Memestra uses a cache to speedup some computations. To handle the cache, memestra installs a tool named ``memestra-cache``.
+*Memestra* uses two caches to speedup some computations.
 
-Memestra's caching infrastructure is a file-based cache, located in home/<user>/.memestra (RW) and in <prefix>/memestra/cache (RO). The key is a hash of the file content and the value is deprecation information, generator used, etc.
+One is a declarative cache, installed in ``<prefix>/share/memestra``. The second
+one is an automatic cache, installed in ``<home>/.memestra``.
 
-There are two kind of keys: recursive and non-recursive. The recursive one also uses the hash of imported modules, so that if an imported module changes, the hash of the importing module aslo changes.
+Declarative Cache
+-----------------
 
-To interact with memestra cache:
+The declarative cache is a file-based cache manually managed by users or
+third-party packages. Its structure is very simple: to describe a Python file
+whose path is ``<root>/numpy/random/__init__.py``, one need to drop a file in
+``<prefix>/share/memestra/numpy/random/__init__.yml``. The yaml content looks
+like the following:
+
+.. code:: yaml
+
+    deprecated: ['deprecated_function1', 'deprecated_function2:some reason']
+    generator: manual
+    name: numpy.random
+    version: 1
+
+- The ``deprecated`` field is the most important one. It contains a list of
+  strings, each element being the name of a deprecated identifier. Text after
+  the first (optional) ``:`` is usead as deprectation documentation.
+
+- The ``generator`` field must be set to ``manual``.
+
+- The ``name`` field is informative, it documents the cache entry and is usually
+  set to the entry path.
+
+- The ``version`` field is used to track compatibility with further format
+  changes. Current value is ``1``.
+
+When hitting an entry in the declarative cache, *memestra* does **not**
+process the content of the file, and uses the entry content instead.
+
+
+Automatic Cache
+---------------
+
+To avoid redundant computations, *memestra* also maintains a cache of the visited file
+and the associated deprecation results.
+
+To handle the cache, *memestra* provides a tool named ``memestra-cache``.
+
+*Memestra*'s caching infrastructure is a file-based cache, located in
+home/<user>/.memestra (RW). The key is a
+hash of the file content and the value contains deprecation information, generator
+used, etc.
+
+There are two kind of keys: recursive and non-recursive. The recursive one also
+uses the hash of imported modules, so that if an imported module changes, the
+hash of the importing module also changes.
+
+To interact with *memestra* caches:
 
 **Positional arguments:**
 
 ``-set``
 
-Set a cache entry
+Set a cache entry in the automatic cache
 
 ``-list``
 
-List cache entries
+List cache entries for both caches
 
 ``-clear``
 
-Remove all cache entries
+Remove all cache entries from the automatic cache
 
 ``-docparse``
 
-Set cache entry from docstring
+Set cache entry from docstring in the automatic cache
 
 
 **Optional arguments:**

memestra/caching.py

@@ -1,5 +1,6 @@
 import os
 import hashlib
+import sys
 import yaml
 
 # not using gast because we only rely on Import and ImportFrom, which are
@@ -131,12 +132,15 @@ def __init__(self, keycls):
         self.keycls = keycls
         self.created = dict()
 
-    def __call__(self, module_path):
+    def __call__(self, module_path, name_hint=None ):
         if module_path in self.created:
             return self.created[module_path]
         else:
             self.created[module_path] = None  # creation in process
             key = self.keycls(module_path, self)
+            if name_hint is None:
+                name_hint = os.path.splitext(os.path.basename(module_path))[0]
+            key.name = name_hint
             self.created[module_path] = key
             return key
 
@@ -153,12 +157,16 @@ class CacheKeyFactory(CacheKeyFactoryBase):
     class CacheKey(object):
 
         def __init__(self, module_path, _):
-            self.name, _ = os.path.splitext(os.path.basename(module_path))
+
             with open(module_path, 'rb') as fd:
                 module_content = fd.read()
                 module_hash = hashlib.sha256(module_content).hexdigest()
                 self.module_hash = module_hash
 
+        @property
+        def path(self):
+            return self.name.replace('.', os.path.sep)
+
     def __init__(self):
         super(CacheKeyFactory, self).__init__(CacheKeyFactory.CacheKey)
 
@@ -176,7 +184,6 @@ class CacheKey(object):
         def __init__(self, module_path, factory):
             assert module_path not in factory.created or factory.created[module_path] is None
 
-            self.name, _ = os.path.splitext(os.path.basename(module_path))
             with open(module_path, 'rb') as fd:
                 module_content = fd.read()
 
@@ -204,13 +211,51 @@ def __init__(self, module_path, factory):
 
                 self.module_hash = hashlib.sha256("".join(hashes).encode("ascii")).hexdigest()
 
+        @property
+        def path(self):
+            return self.name.replace('.', os.path.sep)
+
     def __init__(self):
         super(RecursiveCacheKeyFactory, self).__init__(RecursiveCacheKeyFactory.CacheKey)
 
 
+class SharedCache(object):
+
+    def __init__(self):
+        shared_dir = os.path.join(sys.prefix, 'share', 'memestra')
+        os.makedirs(shared_dir, exist_ok=True)
+        self.cache_entries = {}
+
+        for root, dirs, files in os.walk(shared_dir):
+            for fname in files:
+                if not fname.endswith('.yml'):
+                    continue
+                # Not loading all entries on startup,
+                # doing it lazily upon __getitem__
+
+                if fname == '__init__.yml':
+                    key = root[1 + len(shared_dir):]
+                else:
+                    key = os.path.join(root[1 + len(shared_dir):], fname[:-4])
+                self.cache_entries[key] = os.path.join(root, fname)
+
+    def __contains__(self, key):
+        return key in self.cache_entries
+
+    def __getitem__(self, key):
+        cache_path = self.cache_entries[key]
+        with open(cache_path, 'r') as yaml_fd:
+            return yaml.load(yaml_fd, Loader=yaml.SafeLoader)
+
+    def keys(self):
+        return self.cache_entries.keys()
+
+
 class Cache(object):
 
     def __init__(self, cache_dir=None):
+        self.shared_cache = SharedCache()
+
         if cache_dir is not None:
             self.cachedir = cache_dir
         else:
@@ -223,17 +268,20 @@ def __init__(self, cache_dir=None):
                 memestra_dir = 'memestra'
             self.cachedir = os.path.expanduser(os.path.join(user_config_dir,
                                                             memestra_dir))
-
         os.makedirs(self.cachedir, exist_ok=True)
 
     def _get_path(self, key):
         return os.path.join(self.cachedir, key.module_hash)
 
     def __contains__(self, key):
+        if key.path in self.shared_cache:
+            return True
         cache_path = self._get_path(key)
         return os.path.isfile(cache_path)
 
     def __getitem__(self, key):
+        if key.path in self.shared_cache:
+            return self.shared_cache[key.path]
         cache_path = self._get_path(key)
         with open(cache_path, 'r') as yaml_fd:
             return yaml.load(yaml_fd, Loader=yaml.SafeLoader)
@@ -255,6 +303,11 @@ def items(self):
             with open(cache_path, 'r') as yaml_fd:
                 yield key, yaml.load(yaml_fd, Loader=yaml.SafeLoader)
 
+
+    def shared_items(self):
+        for key in self.shared_cache.keys():
+            yield key, self.shared_cache[key]
+
     def clear(self):
         count = 0
         for key in self.keys():
@@ -278,8 +331,15 @@ def run_set(args):
 
 def run_list(args):
     cache = Cache(cache_dir=args.cache_dir)
-    for k, v in cache.items():
+    print('declarative cache')
+    print('-----------------')
+    for k, v in cache.shared_items():
         print('{}: {} ({})'.format(k, v['name'], len(v['deprecated'])))
+    print()
+    print('automatic cache')
+    print('---------------')
+    for k, v in cache.items():
+        print('{}: {} ({})'.format(k[:16], v['name'], len(v['deprecated'])))
 
 
 def run_clear(args):

memestra/memestra.py

@@ -106,7 +106,7 @@ def load_deprecated_from_module(self, module_name, level=None):
         if module_path is None:
             return None
 
-        module_key = self.key_factory(module_path)
+        module_key = self.key_factory(module_path, name_hint=module_name)
 
         # either find it in the cache
         if module_key in self.cache:

tests/share/memestra/gast/__init__.yml

@@ -0,0 +1,4 @@
+deprecated: ['parse']
+generator: manual
+name: gast.__init__
+version: 1

tests/share/memestra/gast/astn.yml

@@ -0,0 +1,4 @@
+deprecated: ['AstToGAst:because']
+generator: manual
+name: gast.astn
+version: 1

tests/test_imports.py

@@ -1,4 +1,4 @@
-from unittest import TestCase
+from unittest import TestCase, mock
 from textwrap import dedent
 from io import StringIO
 import memestra
@@ -235,3 +235,19 @@ def test_import_pkg_level_star(self):
         self.checkDeprecatedUses(
             code,
             [('foo', '<>', 2, 0, 'why'), ('foo', '<>', 4, 4, 'why')])
+
+    def test_shared_cache(self):
+        # We have a fake description for gast in tests/share/memestra
+        # Setup the shared cache to use it.
+        with mock.patch('sys.prefix', os.path.dirname(__file__)):
+            self.checkDeprecatedUses(
+                'from gast import parse',
+                [('parse', '<>', 1, 0, None)])
+
+    def test_shared_cache_sub(self):
+        # We have a fake description for gast in tests/share/memestra
+        # Setup the shared cache to use it.
+        with mock.patch('sys.prefix', os.path.dirname(__file__)):
+            self.checkDeprecatedUses(
+                'from gast.astn import AstToGAst',
+                [('AstToGAst', '<>', 1, 0, 'because')])