feat: Added support for variety of options for source_path, closes claranet#12 (claranet#25)

Author: ahlinc

Diff for: README.md

@@ -17,7 +17,7 @@ Not supported, yet:
 
 This Terraform module is the part of [serverless.tf framework](https://github.com/antonbabenko/serverless.tf), which aims to simplify all operations when working with the serverless in Terraform:
 
-1. Build and install dependencies - [read more](#build).
+1. Build and install dependencies - [read more](#build). Requires Python 3.6 or newer.
 2. Create, store, and use deployment packages - [read more](#package).
 3. Create, update, and publish AWS Lambda Function and Lambda Layer - [see usage](#usage).
 4. Create static and dynamic aliases for AWS Lambda Function - [see usage](#usage), see [modules/alias](https://github.com/terraform-aws-modules/terraform-aws-lambda/tree/master/modules/alias).
@@ -306,7 +306,7 @@ module "lambda" {
 
 This is one of the most complicated part done by the module and normally you don't have to know internals.
 
-`package.py` is Python script which does it. Make sure, Python 3.7 or newer is installed. The main functions of the script are to generate a filename of zip-archive based on the content of the files, verify if zip-archive has been already created, and create zip-archive only when it is necessary (during `apply`, not `plan`).
+`package.py` is Python script which does it. Make sure, Python 3.6 or newer is installed. The main functions of the script are to generate a filename of zip-archive based on the content of the files, verify if zip-archive has been already created, and create zip-archive only when it is necessary (during `apply`, not `plan`).
 
 Hash of zip-archive created with the same content of the files is always identical which prevents unnecessary force-updates of the Lambda resources unless content modifies. If you need to have different filenames for the same content you can specify extra string argument `hash_extra`.
 
@@ -315,7 +315,7 @@ When calling this module multiple times in one execution to create packages with
 
 ## <a name="build"></a> Debug
 
-Building and packaging has been historically hard to debug (especially with Terraform), so we made an effort to make it easier for user to see debug info. There are 3 different debug levels: `DEBUG` - to see only what is happening during planning phase, `DEBUG2` - to see all logging values, `DEBUG3` - to see all logging values and env variables (be careful sharing your env variables as they may contain secrets!).
+Building and packaging has been historically hard to debug (especially with Terraform), so we made an effort to make it easier for user to see debug info. There are 3 different debug levels: `DEBUG` - to see only what is happening during planning phase and how a zip file content filtering in case of applied patterns, `DEBUG2` - to see more logging output, `DEBUG3` - to see all logging values, `DUMP_ENV` - to see all logging values and env variables (be careful sharing your env variables as they may contain secrets!).
 
 User can specify debug level like this:
 

Diff for: package.py

@@ -2,8 +2,8 @@
 
 import sys
 
-if sys.version_info < (3, 7):
-    raise RuntimeError("A python version 3.7 or newer is required")
+if sys.version_info < (3, 6):
+    raise RuntimeError("A python version 3.6 or newer is required")
 
 import os
 import re
@@ -26,12 +26,15 @@
 import logging
 
 PY38 = sys.version_info >= (3, 8)
+PY37 = sys.version_info >= (3, 7)
+PY36 = sys.version_info >= (3, 6)
 
 ################################################################################
 # Logging
 
 DEBUG2 = 9
 DEBUG3 = 8
+DUMP_ENV = 1
 
 log_handler = None
 log = logging.getLogger()
@@ -43,6 +46,7 @@ def configure_logging(use_tf_stderr=False):
 
     logging.addLevelName(DEBUG2, 'DEBUG2')
     logging.addLevelName(DEBUG3, 'DEBUG3')
+    logging.addLevelName(DUMP_ENV, 'DUMP_ENV')
 
     class LogFormatter(logging.Formatter):
         default_format = '%(message)s'
@@ -139,28 +143,29 @@ def list_files(top_path, log=None):
     return results
 
 
-def dataclass(name, **fields):
-    typ = type(name, (object,), {
-        '__slots__': fields.keys(),
-        '__getattr__': lambda *_: None,
+def dataclass(name):
+    typ = type(name, (dict,), {
+        '__getattr__': lambda self, x: self.get(x),
+        '__init__': lambda self, **k: self.update(k),
     })
-    for k, v in fields.items():
-        setattr(typ, k, v)
     return typ
 
 
 def datatree(name, **fields):
-    def decode_json(v):
+    def decode_json(k, v):
         if v and isinstance(v, str) and v[0] in '"[{':
             try:
-                return json.loads(v)
+                o = json.loads(v)
+                if isinstance(o, dict):
+                    return dataclass(k)(**o)
+                return o
             except json.JSONDecodeError:
                 pass
         return v
 
-    return dataclass(name, **dict(((
-        k, datatree(k, **v) if isinstance(v, dict) else decode_json(v))
-        for k, v in fields.items())))()
+    return dataclass(name)(**dict(((
+        k, datatree(k, **v) if isinstance(v, dict) else decode_json(k, v))
+        for k, v in fields.items())))
 
 
 def timestamp_now_ns():
@@ -291,7 +296,11 @@ def __enter__(self):
         return self.open()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.close(failed=exc_type is not None)
+        if exc_type is not None:
+            self._log.exception("Error during zip archive creation")
+            self.close(failed=True)
+            raise SystemExit(1)
+        self.close()
 
     def _ensure_open(self):
         if self._zip is not None:
@@ -380,10 +389,11 @@ def _write_zinfo(self, zinfo, filename,
             else:
                 zinfo.compress_type = self._compress_type
 
-            if compresslevel is not None:
-                zinfo._compresslevel = compresslevel
-            else:
-                zinfo._compresslevel = self._compresslevel
+            if PY37:
+                if compresslevel is not None:
+                    zinfo._compresslevel = compresslevel
+                else:
+                    zinfo._compresslevel = self._compresslevel
 
         if zinfo.is_dir():
             with zip._lock:
@@ -438,9 +448,9 @@ def _zinfo_from_file(filename, arcname=None, *, strict_timestamps=True):
         isdir = stat.S_ISDIR(st.st_mode)
         mtime = time.localtime(st.st_mtime)
         date_time = mtime[0:6]
-        if not strict_timestamps and date_time[0] < 1980:
+        if strict_timestamps and date_time[0] < 1980:
             date_time = (1980, 1, 1, 0, 0, 0)
-        elif not strict_timestamps and date_time[0] > 2107:
+        elif strict_timestamps and date_time[0] > 2107:
             date_time = (2107, 12, 31, 23, 59, 59)
         # Create ZipInfo instance to store file information
         if arcname is None:
@@ -631,11 +641,19 @@ def pip_requirements_step(path, prefix=None, required=False):
                 hash(requirements)
 
         def commands_step(path, commands):
-            path = os.path.normpath(path)
+            if path:
+                path = os.path.normpath(path)
             batch = []
             for c in commands:
                 if isinstance(c, str):
                     if c.startswith(':zip'):
+                        if path:
+                            hash(path)
+                        else:
+                            # If path doesn't defined for a block with
+                            # commands it will be set to Terraform's
+                            # current working directory
+                            path = query.paths.cwd
                         if batch:
                             step('sh', path, '\n'.join(batch))
                             batch.clear()
@@ -644,15 +662,18 @@ def commands_step(path, commands):
                             _, _path, prefix = c
                             prefix = prefix.strip()
                             _path = os.path.normpath(os.path.join(path, _path))
-                            step('zip', _path, prefix)
+                            step('zip:embedded', _path, prefix)
+                        elif len(c) == 2:
+                            prefix = None
+                            _, _path = c
+                            step('zip:embedded', _path, prefix)
                         elif len(c) == 1:
                             prefix = None
-                            step('zip', path, prefix)
+                            step('zip:embedded', path, prefix)
                         else:
                             raise ValueError(
-                                ':zip command can have zero '
-                                'or 2 arguments: {}'.format(c))
-                        hash(path)
+                                ":zip invalid call signature, use: "
+                                "':zip [path [prefix_in_zip]]'")
                     else:
                         batch.append(c)
 
@@ -706,30 +727,30 @@ def execute(self, build_plan, zip_stream, query):
 
         for action in build_plan:
             cmd = action[0]
-            if cmd == 'zip':
+            if cmd.startswith('zip'):
+                ts = 0 if cmd == 'zip:embedded' else None
                 source_path, prefix = action[1:]
                 if sh_work_dir:
                     if source_path != sh_work_dir:
                         source_path = sh_work_dir
-                if pf:
-                    for path in pf.filter(source_path, prefix):
-                        if os.path.isdir(source_path):
-                            arcname = os.path.relpath(path, source_path)
-                        else:
-                            arcname = os.path.basename(path)
-                        zs.write_file(path, prefix, arcname)
+                    if pf:
+                        self._zip_write_with_filter(zs, pf, source_path, prefix,
+                                                    timestamp=ts)
                 else:
                     if os.path.isdir(source_path):
-                        zs.write_dirs(source_path, prefix=prefix)
+                        zs.write_dirs(source_path, prefix=prefix, timestamp=ts)
                     else:
-                        zs.write_file(source_path, prefix=prefix)
+                        zs.write_file(source_path, prefix=prefix, timestamp=ts)
             elif cmd == 'pip':
                 runtime, pip_requirements, prefix = action[1:]
-                with install_pip_requirements(query, zs,
-                                              pip_requirements) as rd:
+                with install_pip_requirements(query, pip_requirements) as rd:
                     if rd:
-                        # XXX: timestamp=0 - what actually do with it?
-                        zs.write_dirs(rd, prefix=prefix, timestamp=0)
+                        if pf:
+                            self._zip_write_with_filter(zs, pf, rd, prefix,
+                                                        timestamp=0)
+                        else:
+                            # XXX: timestamp=0 - what actually do with it?
+                            zs.write_dirs(rd, prefix=prefix, timestamp=0)
             elif cmd == 'sh':
                 r, w = os.pipe()
                 side_ch = os.fdopen(r)
@@ -750,9 +771,22 @@ def execute(self, build_plan, zip_stream, query):
             elif cmd == 'clear:filter':
                 pf = None
 
+    @staticmethod
+    def _zip_write_with_filter(zip_stream, path_filter, source_path, prefix,
+                               timestamp=None):
+        for path in path_filter.filter(source_path, prefix):
+            if os.path.isdir(source_path):
+                arcname = os.path.relpath(path, source_path)
+            else:
+                arcname = os.path.basename(path)
+            zip_stream.write_file(path, prefix, arcname, timestamp=timestamp)
+
 
 @contextmanager
-def install_pip_requirements(query, zip_stream, requirements_file):
+def install_pip_requirements(query, requirements_file):
+    # TODO:
+    #  1. Emit files instead of temp_dir
+
     if not os.path.exists(requirements_file):
         yield
         return
@@ -892,10 +926,15 @@ def prepare_command(args):
     # Load the query.
     query_data = json.load(sys.stdin)
 
-    if log.isEnabledFor(DEBUG3):
+    if log.isEnabledFor(DUMP_ENV):
         log.debug('ENV: %s', json.dumps(dict(os.environ), indent=2))
     if log.isEnabledFor(DEBUG2):
-        log.debug('QUERY: %s', json.dumps(query_data, indent=2))
+        if log.isEnabledFor(DEBUG3):
+            log.debug('QUERY: %s', json.dumps(query_data, indent=2))
+        else:
+            log_excludes = ('source_path', 'hash_extra_paths', 'paths')
+            qd = {k: v for k, v in query_data.items() if k not in log_excludes}
+            log.debug('QUERY (excerpt): %s', json.dumps(qd, indent=2))
 
     query = datatree('prepare_query', **query_data)
 
@@ -944,7 +983,6 @@ def prepare_command(args):
     build_data = {
         'filename': filename,
         'runtime': runtime,
-        'source_path': source_path,
         'artifacts_dir': artifacts_dir,
         'build_plan': build_plan,
     }
@@ -1050,7 +1088,7 @@ def zip_cmd(args):
                 subprocess.call([zipinfo, args.zipfile])
             log.debug('-' * 80)
             log.debug('Source code hash: %s',
-                         source_code_hash(open(args.zipfile, 'rb').read()))
+                      source_code_hash(open(args.zipfile, 'rb').read()))
 
     p = hidden_parser('zip', help='Zip folder with provided files timestamp')
     p.set_defaults(command=zip_cmd)