Merge pull request #28 from IdentityPython/external_storage

External storage

Author: rohe

.travis.yml

@@ -1,3 +1,4 @@
+dist: bionic
 sudo: false
 language: python
 python:

setup.py

@@ -61,7 +61,7 @@ def run_tests(self):
         "Programming Language :: Python :: 3.8",
         "Topic :: Software Development :: Libraries :: Python Modules"],
     install_requires=[
-        "cryptojwt>=1.1.0",
+        "cryptojwt>=1.5.0",
         "pyOpenSSL",
         "filelock>=3.0.12"
     ],

src/oidcmsg/__init__.py

@@ -1,5 +1,5 @@
 __author__ = 'Roland Hedberg'
-__version__ = '1.1.4'
+__version__ = '1.2.0'
 
 import os
 

src/oidcmsg/context.py

@@ -4,10 +4,8 @@
 from cryptojwt import KeyJar
 from cryptojwt.key_jar import init_key_jar
 
+from oidcmsg.impexp import ImpExp
 from oidcmsg.message import Message
-from oidcmsg.storage.init import get_storage_conf
-from oidcmsg.storage.init import init_storage
-from oidcmsg.storage.init import storage_factory
 
 
 def add_issuer(conf, issuer):
@@ -22,44 +20,24 @@ def add_issuer(conf, issuer):
     return res
 
 
-class OidcContext:
+class OidcContext(ImpExp):
+    parameter = {"keyjar": KeyJar, "issuer": None}
+
     def __init__(self, config=None, keyjar=None, entity_id=''):
+        ImpExp.__init__(self)
         if config is None:
             config = {}
 
-        self.db_conf = config.get('db_conf')
-        if self.db_conf:
-            _iss = config.get('issuer')
-            if _iss:
-                self.db_conf = add_issuer(self.db_conf, _iss)
-
-            if self.db_conf.get('default'):
-                self.db = init_storage(self.db_conf)
-            else:
-                self.db = init_storage()
-        else:
-            self.db = init_storage()
-
-        self.keyjar = self._keyjar(keyjar, self.db_conf, config, entity_id=entity_id)
-
-    def add_boxes(self, boxes, db_conf):
-        for key, attr in boxes.items():
-            setattr(self, attr, init_storage(db_conf, key))
+        self.issuer = entity_id
+        self.keyjar = self._keyjar(keyjar, conf=config, entity_id=entity_id)
 
-    def _keyjar(self, keyjar=None, db_conf=None, conf=None, entity_id=''):
+    def _keyjar(self, keyjar=None, conf=None, entity_id=''):
         if keyjar is None:
-            _storage = None
-            if db_conf:
-                _cnf = get_storage_conf(db_conf, 'keyjar')
-                if _cnf:
-                    _storage = storage_factory(_cnf)
-
             if 'keys' in conf:
                 args = {k: v for k, v in conf["keys"].items() if k != "uri_path"}
-                args.update({'storage': _storage})
                 _keyjar = init_key_jar(**args)
             else:
-                _keyjar = KeyJar(storage=_storage)
+                _keyjar = KeyJar()
                 if 'jwks' in conf:
                     _keyjar.import_jwks(conf['jwks'], '')
 
@@ -74,15 +52,3 @@ def _keyjar(self, keyjar=None, db_conf=None, conf=None, entity_id=''):
             return _keyjar
         else:
             return keyjar
-
-    def set(self, item, value):
-        if isinstance(value, Message):
-            self.db[item] = value.to_dict()
-        else:
-            self.db[item] = value
-
-    def get(self, item):
-        if item == 'seed':
-            return bytes(self.db[item], 'utf-8')
-        else:
-            return self.db[item]

src/oidcmsg/impexp.py

@@ -0,0 +1,89 @@
+from typing import List
+from typing import Optional
+
+from cryptojwt.utils import importer
+from cryptojwt.utils import qualified_name
+
+from oidcmsg.message import Message
+
+
+class ImpExp:
+    parameter = {}
+
+    def __init__(self):
+        pass
+
+    def _dump(self, cls, item, exclude_attributes: Optional[List[str]] = None) -> dict:
+        if cls in [None, "", [], {}]:
+            val = item
+        elif isinstance(item, Message):
+            val = {qualified_name(item.__class__): item.to_dict()}
+        elif cls == object:
+            val = qualified_name(item)
+        elif isinstance(cls, list):
+            val = [self._dump(cls[0], v, exclude_attributes) for v in item]
+        else:
+            val = item.dump(exclude_attributes=exclude_attributes)
+
+        return val
+
+    def dump(self, exclude_attributes: Optional[List[str]] = None) -> dict:
+        _exclude_attributes = exclude_attributes or []
+        info = {}
+        for attr, cls in self.parameter.items():
+            if attr in _exclude_attributes:
+                continue
+
+            item = getattr(self, attr, None)
+            if item is None:
+                continue
+
+            info[attr] = self._dump(cls, item, exclude_attributes)
+
+        return info
+
+    def _local_adjustments(self):
+        pass
+
+    def _load(self, cls, item):
+        if cls in [None, "", [], {}]:
+            val = item
+        elif cls == object:
+            val = importer(item)
+        elif isinstance(cls, list):
+            val = [cls[0]().load(v) for v in item]
+        elif issubclass(cls, Message):
+            val = cls().from_dict(item)
+        else:
+            val = cls().load(item)
+
+        return val
+
+    def load(self, item: dict):
+        for attr, cls in self.parameter.items():
+            if attr not in item:
+                continue
+
+            setattr(self, attr, self._load(cls, item[attr]))
+
+        self._local_adjustments()
+        return self
+
+    def flush(self):
+        """
+        Reset the content of the instance to its pristine state
+
+        :return: A reference to the instance itself
+        """
+        for attr, cls in self.parameter.items():
+            if cls is None:
+                setattr(self, attr, None)
+            elif cls == "":
+                setattr(self, attr, "")
+            elif cls == []:
+                setattr(self, attr, [])
+            elif cls == {}:
+                setattr(self, attr, {})
+            else:
+                setattr(self, attr, None)
+        return self

src/oidcmsg/storage/abfile.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import time
+from urllib.parse import quote_plus
 
 from filelock import FileLock
 
@@ -23,7 +24,7 @@ class AbstractFileSystem(Storage):
     Not directories in directories.
     """
 
-    def __init__(self, conf_dict):
+    def __init__(self, conf_dict, ):
         """
         items = FileSystem(
             {
@@ -44,15 +45,12 @@ def __init__(self, conf_dict):
 
         super().__init__(conf_dict)
         self.config = conf_dict
-        _fdir = conf_dict.get('fdir', '')
-        if '{issuer}' in _fdir:
-            issuer = conf_dict.get('issuer')
-            if not issuer:
-                raise ValueError('Missing issuer value')
-            self.fdir = _fdir.format(issuer=issuer)
-        else:
-            self.fdir = _fdir
 
+        _fdir = conf_dict.get('fdir', '.')
+        if "issuer" in conf_dict:
+            _fdir = os.path.join(_fdir, quote_plus(conf_dict["issuer"]))
+
+        self.fdir = _fdir
         self.fmtime = {}
         self.storage = {}
 
@@ -87,11 +85,13 @@ def __getitem__(self, item):
         :return:
         """
         item = self.key_conv.serialize(item)
-        if self._is_file(item):
-            if self.is_changed(item):
-                logger.info("File content change in {}".format(item))
-                fname = os.path.join(self.fdir, item)
-                self.storage[item] = self._read_info(fname)
+        fname = os.path.join(self.fdir, item)
+        if self._is_file(fname):
+            lock = FileLock('{}.lock'.format(fname))
+            with lock:
+                if self.is_changed(item, fname):
+                    logger.info("File content change in {}".format(item))
+                    self.storage[item] = self._read_info(fname)
 
             logger.debug('Read from "%s"', item)
             return self.storage[item]
@@ -165,19 +165,17 @@ def get_mtime(fname):
 
         return mtime
 
-    def _is_file(self, item):
-        fname = os.path.join(self.fdir, item)
+    def _is_file(self, fname):
         return os.path.isfile(fname)
 
-    def is_changed(self, item):
+    def is_changed(self, item, fname):
         """
         Find out if this item has been modified since last.
         When I get here I know that item points to an existing file.
 
         :param item: A key
         :return: True/False
         """
-        fname = os.path.join(self.fdir, item)
         mtime = self.get_mtime(fname)
 
         try:
@@ -223,7 +221,7 @@ def synch(self):
                 continue
 
             if f in self.fmtime:
-                if self.is_changed(f):
+                if self.is_changed(f, fname):
                     self.storage[f] = self._read_info(fname)
             else:
                 mtime = self.get_mtime(fname)

src/oidcmsg/storage/converter.py

@@ -12,8 +12,8 @@ def deserialize(self, str):
 
 
 class JSON:
-    def serialize(self, str):
-        return json.dumps(str)
+    def serialize(self, item):
+        return json.dumps(item)
 
     def deserialize(self, str):
         return json.loads(str)