Merge pull request #114 from declension/issue-113-ignores-are-regexes

Quote exclusion regexes properly (#113)

Author: jarosser06

README.md

@@ -17,14 +17,19 @@ python setup.py install
 
 ### Configuration File
 The lambda uploader expects a directory with, at a minimum, your lambda function
-and a lambda.json file.  It is not necessary to set requirements in your config
+and a `lambda.json` file.  It is not necessary to set requirements in your config
 file since the lambda uploader will also check for and use a requirements.txt file.
 
-Please note that you can leave the "vpc" object out of your config if you want your
+Please note that you can leave the `vpc` object out of your config if you want your
 lambda function to use your default VPC and subnets. If you wish to use your lambda
 function inside a specific VPC, make sure you set up the role correctly to allow this.
 
-Example lambda.json file:
+Note also the `ignore` entry is an array of regular expression strings
+used to match against the relative paths - be careful to quote accordingly.
+For example, a traditional `*.txt` "glob" is matched by the JSON string: 
+`".*\\.txt$"` (or just `"\\.txt$"`).
+
+Example `lambda.json` file:
 ```json
 {
   "name": "myFunction",
@@ -35,9 +40,9 @@ Example lambda.json file:
   "role": "arn:aws:iam::00000000000:role/lambda_basic_execution",
   "requirements": ["pygithub"],
   "ignore": [
-    "circle.yml",
-    ".git",
-    "/*.pyc"
+    "circle\\.yml$",
+    "\\.git$",
+    "/.*\\.pyc$"
   ],
   "timeout": 30,
   "memory": 512,

example/lambda.json

@@ -6,9 +6,9 @@
   "role": "arn:aws:iam::00000000000:role/lambda_basic_execution",
   "requirements": ["Jinja2==2.8"],
   "ignore": [
-    "circle.yml",
-    ".git",
-    "/*.pyc"
+    "circle\\.yml$",
+    "\\.git$",
+    "/.*\\.pyc$"
   ],
   "timeout": 30,
   "memory": 512

lambda_uploader/package.py

@@ -17,12 +17,13 @@
 import zipfile
 import logging
 import sys
+import re
 
 from subprocess import Popen, PIPE
 from lambda_uploader import utils
 from distutils.spawn import find_executable
 
-# Python 2/3 compatability
+# Python 2/3 compatibility
 try:
     basestring
 except NameError:
@@ -34,8 +35,8 @@
 ZIPFILE_NAME = 'lambda_function.zip'
 
 
-def build_package(path, requires, virtualenv=None, ignore=[],
-                  extra_files=[], zipfile_name=ZIPFILE_NAME):
+def build_package(path, requires, virtualenv=None, ignore=None,
+                  extra_files=None, zipfile_name=ZIPFILE_NAME):
     '''Builds the zip file and creates the package with it'''
     pkg = Package(path, zipfile_name)
 
@@ -69,7 +70,7 @@ def __init__(self, path, zipfile_name=ZIPFILE_NAME):
         self._requirements_file = os.path.join(self._path, "requirements.txt")
         self._extra_files = []
 
-    def build(self, ignore=[]):
+    def build(self, ignore=None):
         '''Calls all necessary methods to build the Lambda Package'''
         self._prepare_workspace()
         self.install_dependencies()
@@ -214,7 +215,7 @@ def _install_requirements(self):
             if prc.returncode is not 0:
                 raise Exception('pip returned unsuccessfully')
 
-    def package(self, ignore=[]):
+    def package(self, ignore=None):
         """
         Create a zip file of the lambda script and its dependencies.
 
@@ -223,6 +224,7 @@ def package(self, ignore=[]):
             those files when creating the zip file. The paths to be matched are
             local to the source root.
         """
+        ignore = ignore or []
         package = os.path.join(self._temp_workspace, 'lambda_package')
 
         # Copy site packages into package base
@@ -243,13 +245,13 @@ def package(self, ignore=[]):
                 utils.copy_tree(lib64_path, package)
 
         # Append the temp workspace to the ignore list:
-        ignore += ["^%s/*" % TEMP_WORKSPACE_NAME]
+        ignore.append(r"^%s/.*" % re.escape(TEMP_WORKSPACE_NAME))
         utils.copy_tree(self._path, package, ignore)
 
         # Add extra files
         for p in self._extra_files:
             LOG.info('Copying extra %s into package' % p)
-            ignore += ["%s" % p]
+            ignore.append(re.escape(p))
             if os.path.isdir(p):
                 utils.copy_tree(p, package, ignore=ignore, include_parent=True)
             else:

lambda_uploader/utils.py

@@ -21,7 +21,8 @@
 LOG = logging.getLogger(__name__)
 
 
-def copy_tree(src, dest, ignore=[], include_parent=False):
+def copy_tree(src, dest, ignore=None, include_parent=False):
+    ignore = ignore or []
     if os.path.isfile(src):
         raise Exception('Cannot use copy_tree with a file as the src')
 
@@ -55,7 +56,8 @@ def copy_tree(src, dest, ignore=[], include_parent=False):
 
 # Iterate through every item in ignore
 # and check for matches in the path
-def _ignore_file(path, ignore=[]):
+def _ignore_file(path, ignore=None):
+    ignore = ignore or []
     if not ignore:
         return False
     for ign in ignore:

tests/extra/.dotfile

@@ -0,0 +1 @@
+# Nothing here

tests/test_package.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import sys
 import pytest
 
@@ -9,6 +10,7 @@
 TESTING_TEMP_DIR = '.testing_temp'
 WORKING_TEMP_DIR = path.join(TESTING_TEMP_DIR, '.lambda_uploader_temp')
 PACKAGE_TEMP_DIR = path.join(WORKING_TEMP_DIR, 'lambda_package')
+DOTFILE_REGEX = r'^\.[^.].*'
 
 
 def setup_module(module):
@@ -21,6 +23,10 @@ def teardown_module(module):
     rmtree(TESTING_TEMP_DIR)
 
 
+def setup():
+    shutil.rmtree(PACKAGE_TEMP_DIR, True)
+
+
 def test_package_zip_location():
     pkg = package.Package(TESTING_TEMP_DIR)
     assert pkg.zip_file == '.testing_temp/lambda_function.zip'
@@ -139,6 +145,40 @@ def test_package_with_extras():
     expected_extra_file2 = path.join(PACKAGE_TEMP_DIR, 'extra/foo/__init__.py')
     assert path.isfile(expected_extra_file2)
 
+    # test a hidden file
+    expected_dotfile = path.join(PACKAGE_TEMP_DIR, 'extra', '.dotfile')
+    assert path.isfile(expected_dotfile)
+
+
+def test_package_with_ignores():
+    pkg = package.Package(TESTING_TEMP_DIR)
+    pkg.extra_file(path.join('tests', 'extra'))
+    pkg.package(ignore=[DOTFILE_REGEX])
+
+    # test ignored file is *not* there
+    dotfile = path.join(PACKAGE_TEMP_DIR, 'extra', '.dotfile')
+    assert not path.exists(dotfile)
+
+
+def test_ignores_using_all_items_and_regex():
+    pkg = package.Package(TESTING_TEMP_DIR)
+    pyc_path = path.join(TESTING_TEMP_DIR, 'fake.pyc')
+    py_path = path.join(TESTING_TEMP_DIR, 'real.py')
+    open(py_path, 'w').close()
+    open(pyc_path, 'w').close()
+
+    pkg.package([r"dummy.*", r'[a-z]+\.pyc'])
+
+    os.remove(py_path)
+    os.remove(pyc_path)
+
+    # test the ignores has excluded the .pyc
+    expected_extra_file = path.join(PACKAGE_TEMP_DIR, 'fake.pyc')
+    assert not path.exists(expected_extra_file)
+
+    # ...but the path not affected by either ignore entry remains
+    assert path.exists(path.join(PACKAGE_TEMP_DIR, 'real.py'))
+
 
 def test_package_name():
     pkg = package.Package(TESTING_TEMP_DIR, zipfile_name='test.zip')

tests/test_utils.py

@@ -19,14 +19,19 @@
 ]
 
 TEST_IGNORE = [
-    'ignore/*',
+    'ignore/.*',
     'ignore-me.py',
     # The one below would exclude all the files if files to be ignored were
     # matched based on the full path, rather than the path relative to the
     # source directory, since the source directory contains 'test' in its path.
     'tests',
 ]
 
+IGNORE_TEMP = [
+    r'^\.[^.].*',
+    r'.*\.pyc$'
+]
+
 
 def test_copy_tree():
     os.mkdir(TESTING_TEMP_DIR)
@@ -49,8 +54,25 @@ def test_copy_tree():
 
 
 def test_ignore_file():
-    result = utils._ignore_file('ignore/foo.py', TEST_IGNORE)
-    assert result
+    ignored = utils._ignore_file('ignore/foo.py', TEST_IGNORE)
+    assert ignored
+
+    ignored = utils._ignore_file('ignore', TEST_IGNORE)
+    assert not ignored
+
+    ignored = utils._ignore_file('bar/foo.py', TEST_IGNORE)
+    assert not ignored
+
+
+def test_ignore_file_dotfile():
+    ignored = utils._ignore_file('.mydotfile', IGNORE_TEMP)
+    assert ignored
+
+
+def test_ignore_pyc():
+    ignored = utils._ignore_file('pycharm', IGNORE_TEMP)
+    assert not ignored
+
+    ignored = utils._ignore_file('bar/foo.pyc', IGNORE_TEMP)
+    assert ignored
 
-    res = utils._ignore_file('bar/foo.py', TEST_IGNORE)
-    assert res is False