Use importlib instead of deprecated imp module (#28)

Author: rmanganiello

Diff for: .gitignore

@@ -143,3 +143,6 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
+
+# Test files generated
+tmp*.py

Diff for: awslambdaric/bootstrap.py

@@ -2,23 +2,19 @@
 Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 """
 
+import importlib
 import json
 import logging
 import os
 import sys
 import time
 import traceback
-import warnings
 
 from .lambda_context import LambdaContext
 from .lambda_runtime_client import LambdaRuntimeClient
 from .lambda_runtime_exception import FaultException
 from .lambda_runtime_marshaller import to_json
 
-with warnings.catch_warnings():
-    warnings.filterwarnings("ignore", category=DeprecationWarning)
-    import imp
-
 ERROR_LOG_LINE_TERMINATE = "\r"
 ERROR_LOG_IDENT = "\u00a0"  # NO-BREAK SPACE U+00A0
 
@@ -33,23 +29,14 @@ def _get_handler(handler):
         )
         return make_fault_handler(fault)
 
-    file_handle, pathname, desc = None, None, None
     try:
-        # Recursively loading handler in nested directories
-        for segment in modname.split("."):
-            if pathname is not None:
-                pathname = [pathname]
-            file_handle, pathname, desc = imp.find_module(segment, pathname)
-        if file_handle is None:
-            module_type = desc[2]
-            if module_type == imp.C_BUILTIN:
-                fault = FaultException(
-                    FaultException.BUILT_IN_MODULE_CONFLICT,
-                    "Cannot use built-in module {} as a handler module".format(modname),
-                )
-                request_handler = make_fault_handler(fault)
-                return request_handler
-        m = imp.load_module(modname, file_handle, pathname, desc)
+        if modname.split(".")[0] in sys.builtin_module_names:
+            fault = FaultException(
+                FaultException.BUILT_IN_MODULE_CONFLICT,
+                "Cannot use built-in module {} as a handler module".format(modname),
+            )
+            return make_fault_handler(fault)
+        m = importlib.import_module(modname)
     except ImportError as e:
         fault = FaultException(
             FaultException.IMPORT_MODULE_ERROR,
@@ -66,9 +53,6 @@ def _get_handler(handler):
         )
         request_handler = make_fault_handler(fault)
         return request_handler
-    finally:
-        if file_handle is not None:
-            file_handle.close()
 
     try:
         request_handler = getattr(m, fname)
@@ -402,7 +386,7 @@ def run(app_root, handler, lambda_runtime_api_addr):
             global _GLOBAL_AWS_REQUEST_ID
 
             request_handler = _get_handler(handler)
-        except Exception as e:
+        except Exception:
             error_result = build_fault_result(sys.exc_info(), None)
 
             log_error(error_result, log_sink)

Diff for: requirements/dev.txt

@@ -2,7 +2,7 @@ coverage>=4.4.0
 flake8>=3.3.0
 tox>=2.2.1
 pytest-cov>=2.4.0
-pylint>=1.7.2,<2.0
+pylint>=1.7.2
 black>=20.8b0
 bandit>=1.6.2
 

Diff for: tests/test_bootstrap.py

@@ -2,16 +2,16 @@
 Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 """
 
+import importlib
 import json
 import os
 import re
 import tempfile
 import traceback
 import unittest
-from imp import C_BUILTIN
 from io import StringIO
 from tempfile import NamedTemporaryFile
-from unittest.mock import patch, Mock, MagicMock
+from unittest.mock import MagicMock, Mock, patch
 
 import awslambdaric.bootstrap as bootstrap
 from awslambdaric.lambda_runtime_exception import FaultException
@@ -350,7 +350,7 @@ def __init__(self, message):
 
     def test_handle_event_request_no_module(self):
         def unable_to_import_module(json_input, lambda_context):
-            import invalid_module
+            import invalid_module  # noqa: F401
 
         expected_response = {
             "errorType": "ModuleNotFoundError",
@@ -381,8 +381,8 @@ def unable_to_import_module(json_input, lambda_context):
     def test_handle_event_request_fault_exception(self):
         def raise_exception_handler(json_input, lambda_context):
             try:
-                import invalid_module
-            except ImportError as e:
+                import invalid_module  # noqa: F401
+            except ImportError:
                 raise FaultException(
                     "FaultExceptionType",
                     "Fault exception msg",
@@ -429,8 +429,8 @@ def raise_exception_handler(json_input, lambda_context):
     def test_handle_event_request_fault_exception_logging(self, mock_stdout):
         def raise_exception_handler(json_input, lambda_context):
             try:
-                import invalid_module
-            except ImportError as e:
+                import invalid_module  # noqa: F401
+            except ImportError:
                 raise bootstrap.FaultException(
                     "FaultExceptionType",
                     "Fault exception msg",
@@ -469,8 +469,8 @@ def raise_exception_handler(json_input, lambda_context):
     def test_handle_event_request_fault_exception_logging_notrace(self, mock_stdout):
         def raise_exception_handler(json_input, lambda_context):
             try:
-                import invalid_module
-            except ImportError as e:
+                import invalid_module  # noqa: F401
+            except ImportError:
                 raise bootstrap.FaultException(
                     "FaultExceptionType", "Fault exception msg", None
                 )
@@ -497,8 +497,8 @@ def test_handle_event_request_fault_exception_logging_nomessage_notrace(
     ):
         def raise_exception_handler(json_input, lambda_context):
             try:
-                import invalid_module
-            except ImportError as e:
+                import invalid_module  # noqa: F401
+            except ImportError:
                 raise bootstrap.FaultException("FaultExceptionType", None, None)
 
         bootstrap.handle_event_request(
@@ -523,8 +523,8 @@ def test_handle_event_request_fault_exception_logging_notype_notrace(
     ):
         def raise_exception_handler(json_input, lambda_context):
             try:
-                import invalid_module
-            except ImportError as e:
+                import invalid_module  # noqa: F401
+            except ImportError:
                 raise bootstrap.FaultException(None, "Fault exception msg", None)
 
         bootstrap.handle_event_request(
@@ -549,8 +549,8 @@ def test_handle_event_request_fault_exception_logging_notype_nomessage(
     ):
         def raise_exception_handler(json_input, lambda_context):
             try:
-                import invalid_module
-            except ImportError as e:
+                import invalid_module  # noqa: F401
+            except ImportError:
                 raise bootstrap.FaultException(
                     None,
                     None,
@@ -585,19 +585,16 @@ def raise_exception_handler(json_input, lambda_context):
         self.assertEqual(mock_stdout.getvalue(), error_logs)
 
     @patch("sys.stdout", new_callable=StringIO)
-    @patch("imp.find_module")
-    @patch("imp.load_module")
+    @patch("importlib.import_module")
     def test_handle_event_request_fault_exception_logging_syntax_error(
-        self, mock_load_module, mock_find_module, mock_stdout
+        self, mock_import_module, mock_stdout
     ):
-
         try:
             eval("-")
         except SyntaxError as e:
             syntax_error = e
 
-        mock_find_module.return_value = (None, None, ("", "", None))
-        mock_load_module.side_effect = syntax_error
+        mock_import_module.side_effect = syntax_error
 
         response_handler = bootstrap._get_handler("a.b")
 
@@ -618,7 +615,10 @@ def test_handle_event_request_fault_exception_logging_syntax_error(
 
         sys.stderr.write(mock_stdout.getvalue())
 
-        error_logs = "[ERROR] Runtime.UserCodeSyntaxError: Syntax error in module 'a': unexpected EOF while parsing (<string>, line 1)\r"
+        error_logs = (
+            "[ERROR] Runtime.UserCodeSyntaxError: Syntax error in module 'a': "
+            "unexpected EOF while parsing (<string>, line 1)\r"
+        )
         error_logs += "Traceback (most recent call last):\r"
         error_logs += '  File "<string>" Line 1\r'
         error_logs += "    -\n"
@@ -730,56 +730,57 @@ def test_get_event_handler_import_error(self):
         )
 
     def test_get_event_handler_syntax_error(self):
-        tmp_file = tempfile.NamedTemporaryFile(suffix=".py", dir=".", delete=False)
-        tmp_file.write(
-            b"def syntax_error()\n\tprint('syntax error, no colon after function')"
-        )
-        tmp_file.close()
-        filename_w_ext = os.path.basename(tmp_file.name)
-        filename, _ = os.path.splitext(filename_w_ext)
-        handler_name = "{}.syntax_error".format(filename)
-        response_handler = bootstrap._get_handler(handler_name)
-
-        with self.assertRaises(FaultException) as cm:
-            response_handler()
-        returned_exception = cm.exception
-        self.assertEqual(
-            self.FaultExceptionMatcher(
-                "Syntax error in",
-                "Runtime.UserCodeSyntaxError",
-                ".*File.*\\.py.*Line 1.*",
-            ),
-            returned_exception,
-        )
-        if os.path.exists(tmp_file.name):
-            os.remove(tmp_file.name)
+        importlib.invalidate_caches()
+        with tempfile.NamedTemporaryFile(
+            suffix=".py", dir=".", delete=False
+        ) as tmp_file:
+            tmp_file.write(
+                b"def syntax_error()\n\tprint('syntax error, no colon after function')"
+            )
+            tmp_file.flush()
+
+            filename_w_ext = os.path.basename(tmp_file.name)
+            filename, _ = os.path.splitext(filename_w_ext)
+            handler_name = "{}.syntax_error".format(filename)
+            response_handler = bootstrap._get_handler(handler_name)
+
+            with self.assertRaises(FaultException) as cm:
+                response_handler()
+            returned_exception = cm.exception
+            self.assertEqual(
+                self.FaultExceptionMatcher(
+                    "Syntax error in",
+                    "Runtime.UserCodeSyntaxError",
+                    ".*File.*\\.py.*Line 1.*",
+                ),
+                returned_exception,
+            )
 
     def test_get_event_handler_missing_error(self):
-        tmp_file = tempfile.NamedTemporaryFile(suffix=".py", dir=".", delete=False)
-        tmp_file.write(b"def wrong_handler_name():\n\tprint('hello')")
-        tmp_file.close()
-        filename_w_ext = os.path.basename(tmp_file.name)
-        filename, _ = os.path.splitext(filename_w_ext)
-        handler_name = "{}.my_handler".format(filename)
-        response_handler = bootstrap._get_handler(handler_name)
-        with self.assertRaises(FaultException) as cm:
-            response_handler()
-        returned_exception = cm.exception
-        self.assertEqual(
-            self.FaultExceptionMatcher(
-                "Handler 'my_handler' missing on module '{}'".format(filename),
-                "Runtime.HandlerNotFound",
-            ),
-            returned_exception,
-        )
-        if os.path.exists(tmp_file.name):
-            os.remove(tmp_file.name)
+        importlib.invalidate_caches()
+        with tempfile.NamedTemporaryFile(
+            suffix=".py", dir=".", delete=False
+        ) as tmp_file:
+            tmp_file.write(b"def wrong_handler_name():\n\tprint('hello')")
+            tmp_file.flush()
+
+            filename_w_ext = os.path.basename(tmp_file.name)
+            filename, _ = os.path.splitext(filename_w_ext)
+            handler_name = "{}.my_handler".format(filename)
+            response_handler = bootstrap._get_handler(handler_name)
+            with self.assertRaises(FaultException) as cm:
+                response_handler()
+            returned_exception = cm.exception
+            self.assertEqual(
+                self.FaultExceptionMatcher(
+                    "Handler 'my_handler' missing on module '{}'".format(filename),
+                    "Runtime.HandlerNotFound",
+                ),
+                returned_exception,
+            )
 
-    @patch("imp.find_module")
-    def test_get_event_handler_build_in_conflict(self, mock_find_module):
-        handler_name = "sys.hello"
-        mock_find_module.return_value = (None, None, ("", "", C_BUILTIN))
-        response_handler = bootstrap._get_handler(handler_name)
+    def test_get_event_handler_build_in_conflict(self):
+        response_handler = bootstrap._get_handler("sys.hello")
         with self.assertRaises(FaultException) as cm:
             response_handler()
         returned_exception = cm.exception
@@ -921,7 +922,10 @@ def test_log_error_indentation_standard_log_sink(self, mock_stdout):
         )
         bootstrap.log_error(err_to_log, bootstrap.StandardLogSink())
 
-        expected_logged_error = "[ERROR] ErrorType: Error message\rTraceback (most recent call last):\r\xa0\xa0line1  \r\xa0\xa0line2  \r\xa0\xa0\n"
+        expected_logged_error = (
+            "[ERROR] ErrorType: Error message\rTraceback (most recent call last):"
+            "\r\xa0\xa0line1  \r\xa0\xa0line2  \r\xa0\xa0\n"
+        )
         self.assertEqual(mock_stdout.getvalue(), expected_logged_error)
 
     def test_log_error_indentation_framed_log_sink(self):
@@ -932,7 +936,10 @@ def test_log_error_indentation_framed_log_sink(self):
                 )
                 bootstrap.log_error(err_to_log, log_sink)
 
-            expected_logged_error = "[ERROR] ErrorType: Error message\nTraceback (most recent call last):\n\xa0\xa0line1  \n\xa0\xa0line2  \n\xa0\xa0"
+            expected_logged_error = (
+                "[ERROR] ErrorType: Error message\nTraceback (most recent call last):"
+                "\n\xa0\xa0line1  \n\xa0\xa0line2  \n\xa0\xa0"
+            )
 
             with open(temp_file.name, "rb") as f:
                 content = f.read()
@@ -964,7 +971,10 @@ def test_log_error_empty_stacktrace_line_framed_log_sink(self):
                 )
                 bootstrap.log_error(err_to_log, log_sink)
 
-            expected_logged_error = "[ERROR] ErrorType: Error message\nTraceback (most recent call last):\nline1\n\nline2"
+            expected_logged_error = (
+                "[ERROR] ErrorType: Error message\nTraceback "
+                "(most recent call last):\nline1\n\nline2"
+            )
 
             with open(temp_file.name, "rb") as f:
                 content = f.read()
@@ -1082,11 +1092,10 @@ def test_run(self, mock_runtime_client, mock_handle_event_request):
             MagicMock(),
         ]
 
-        with self.assertRaises(TypeError) as cm:
+        with self.assertRaises(TypeError):
             bootstrap.run(
                 expected_app_root, expected_handler, expected_lambda_runtime_api_addr
             )
-        returned_exception = cm.exception
 
         mock_handle_event_request.assert_called_once()
 
@@ -1108,11 +1117,10 @@ class TestException(Exception):
 
         mock_sys.exit.side_effect = TestException("Boom!")
 
-        with self.assertRaises(TestException) as cm:
+        with self.assertRaises(TestException):
             bootstrap.run(
                 expected_app_root, expected_handler, expected_lambda_runtime_api_addr
             )
-        returned_exception = cm.exception
 
         mock_sys.exit.assert_called_once_with(1)