Merge pull request #12 from CSCfi/feature/typing

add typing and checks for it, move to python 3.8

Author: teemukataja

.github/workflows/style.yml

@@ -8,7 +8,7 @@ jobs:
       max-parallel: 4
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.7]
+        python-version: [3.7, 3.8]
 
     runs-on: ${{ matrix.os }}
 
@@ -24,5 +24,7 @@ jobs:
         pip install tox tox-gh-actions
     - name: Test flake8 syntax with tox
       run: tox -e flake8
+    - name: Test mypy typing with tox
+      run: tox -e mypy
     - name: bandit static check
       run: tox -e bandit

swift_upload_runner/auth.py

@@ -57,7 +57,7 @@ async def handle_login(
 
 async def read_in_keys(
         app: aiohttp.web.Application
-):
+) -> None:
     """Read in keys to the application."""
     keys = os.environ.get("SWIFT_UI_API_AUTH_TOKENS", None)
     app["tokens"] = keys.split(",") if keys is not None else []

swift_upload_runner/common.py

@@ -12,9 +12,9 @@
 
 
 def generate_download_url(
-        host,
-        container=None,
-        object_name=None,
+        host: str,
+        container: typing.Union[str, None] = None,
+        object_name: typing.Union[str, None] = None,
 ) -> str:
     """Generate the download URL to use."""
     if not container and not object_name:
@@ -94,7 +94,7 @@ async def get_upload_instance(
     if p_query:
         query: dict = p_query
     else:
-        query = request.query
+        query = request.query  # type: ignore
 
     # Check the existence of the dictionary structure
     try:
@@ -142,7 +142,9 @@ def get_path_from_list(
     return ret.lstrip("/").rstrip("/")
 
 
-async def handle_delete_preflight(_) -> aiohttp.web.Response:
+async def handle_delete_preflight(
+        _: typing.Union[aiohttp.web.Request, None]
+) -> aiohttp.web.Response:
     """Serve correct response headers to allowed DELETE preflight query."""
     resp = aiohttp.web.Response(
         headers={

swift_upload_runner/download.py

@@ -34,8 +34,8 @@ class FileDownloadProxy:
     def __init__(
             self,
             auth: keystoneauth1.session.Session,
-            chunk_size=128 * 1024
-    ):
+            chunk_size: int = 128 * 1024
+    ) -> None:
         """."""
         # Establish a queue for the proxied file parts
         # Total queue size 128 * 256 * 1024 = 32MiB for now
@@ -103,7 +103,7 @@ def download_into_queue(
             project: str,
             container: str,
             object_name: str
-    ):
+    ) -> None:
         """Download object chunks from stream into the queue."""
         print(f"""
         Downloading from project {project},
@@ -159,7 +159,7 @@ async def a_begin_download(
             project: str,
             container: str,
             object_name: str
-    ):
+    ) -> None:
         """Begin the download process."""
         self.begin_download(
             project,
@@ -172,7 +172,7 @@ def begin_download(
         project: str,
         container: str,
         object_name: str
-    ):
+    ) -> None:
         """Begin the download process."""
         self.t_dload = threading.Thread(
             target=self.download_into_queue,
@@ -188,7 +188,7 @@ def begin_download(
     async def a_write_to_response(
             self,
             resp: aiohttp.web.StreamResponse
-    ):
+    ) -> None:
         """Get the response serving the file."""
         while True:
             chunk = await self.a_read()
@@ -207,7 +207,7 @@ def __init__(
             project: str,
             container: str,
             object_name: str
-    ):
+    ) -> None:
         """."""
         # Initialize the download class
         self.dload = FileDownloadProxy(auth)
@@ -222,7 +222,7 @@ def get_dload(self) -> FileDownloadProxy:
         """Return the specific download instance."""
         return self.dload
 
-    def begin_download(self):
+    def begin_download(self) -> None:
         """Begin download and block until received headers."""
         self.dload.begin_download(
             self.project,
@@ -258,7 +258,7 @@ class TarQueueWrapper:
 
     def __init__(
             self
-    ):
+    ) -> None:
         """."""
         self.q: queue.Queue = queue.Queue(
             maxsize=int(
@@ -269,13 +269,13 @@ def __init__(
     def write(
             self,
             payload: bytes = None
-    ):
+    ) -> None:
         """Emulate BytesIO write function to be used with tarfile."""
         self.q.put(payload)
 
     def read(
             self
-    ):
+    ) -> bytes:
         """Read next chunk."""
         return self.q.get()
 
@@ -287,7 +287,7 @@ def get_q(
 
     async def a_read(
             self
-    ):
+    ) -> bytes:
         """Asynchronously read next chunk."""
         while True:
             try:
@@ -304,8 +304,8 @@ def __init__(
             auth: keystoneauth1.session.Session,
             project: str,
             container: str,
-            chunk_size=128 * 1024
-    ):
+            chunk_size: int = 128 * 1024
+    ) -> None:
         """."""
         self.auth = auth
         self.download_queue: queue.Queue = queue.Queue(
@@ -411,7 +411,7 @@ def _parse_archive_fs(
 
     def get_object_listing(
             self,
-    ):
+    ) -> None:
         """Synchronize the list of objects to download."""
         with requests.get(
                 generate_download_url(
@@ -429,8 +429,8 @@ def get_object_listing(
 
     def sync_folders(
             self,
-            fs
-    ):
+            fs: dict
+    ) -> None:
         """Sycnhronize the folders into the tar archive."""
         if self.archive:
             for i in fs:
@@ -440,15 +440,15 @@ def sync_folders(
 
     def download_init(
             self,
-    ):
+    ) -> None:
         """Create download init."""
         self.download_init_loop(self.fs)
         self.download_queue.put(None)
 
     def download_init_loop(
             self,
-            fs
-    ):
+            fs: dict
+    ) -> None:
         """Loop to run for initializing downloads."""
         if self.archive:
             for i in fs:
@@ -472,7 +472,7 @@ def download_init_loop(
 
     def tar_archiving_loop(
             self,
-    ):
+    ) -> None:
         """Loop to run for initializing tarballing."""
         while True:
             next_file = self.download_queue.get()
@@ -502,20 +502,20 @@ def tar_archiving_loop(
 
     async def a_begin_container_download(
             self,
-    ):
+    ) -> None:
         """Begin the operation for downloading a whole container."""
         self.begin_container_download()
 
     def begin_container_download(
             self,
-    ):
+    ) -> None:
         """Begin the operation for downloading a whole container."""
         self.get_object_listing()
 
         self.archive = tarfile.open(
             name=self.container + ".tar",
             mode="w|",
-            fileobj=self.output_queue
+            fileobj=self.output_queue  # type:ignore
         )
 
         self.sync_folders(self.fs)
@@ -535,7 +535,7 @@ def begin_container_download(
     async def a_write_to_response(
             self,
             response: aiohttp.web.StreamResponse
-    ):
+    ) -> None:
         """Write the tarball into the response."""
         while True:
             chunk = await self.output_queue.a_read()

swift_upload_runner/replicate.py

@@ -34,7 +34,7 @@ def __init__(
             container: str,
             source_project: str,
             source_container: str,
-    ):
+    ) -> None:
         """."""
         self.project = project
         self.container = container
@@ -54,7 +54,7 @@ def __init__(
     async def a_generate_object_from_reader(
             self,
             resp: aiohttp.client.ClientResponse
-    ):
+    ) -> typing.AsyncGenerator:
         """Generate uploaded object chunks from a response."""
         number = 0
         while True:
@@ -67,8 +67,8 @@ async def a_generate_object_from_reader(
 
     async def a_create_container(
             self,
-            segmented=False
-    ):
+            segmented: bool = False
+    ) -> None:
         """Create the container required by the upload."""
         container = \
             f"{self.container}_segments" if segmented else self.container
@@ -91,8 +91,8 @@ async def a_create_container(
 
     async def a_sync_object_segments(
             self,
-            manifest
-    ) -> typing.List[str]:
+            manifest: str
+    ) -> str:
         """Get object segments."""
         async with self.client.get(
                 common.generate_download_url(
@@ -183,8 +183,8 @@ async def a_sync_object_segments(
 
     async def a_copy_object(
             self,
-            object_name
-    ):
+            object_name: str
+    ) -> None:
         """Copy an object from a location."""
         # Get the object stream handle
         async with self.client.get(
@@ -275,7 +275,7 @@ async def a_copy_object(
                         )
                 LOGGER.debug(f"Uploaded manifest for {object_name}")
 
-    async def a_copy_from_container(self):
+    async def a_copy_from_container(self) -> None:
         """Copy objects from a source container."""
         LOGGER.debug(
             f"Fetching objects from container {self.source_container}"
@@ -285,6 +285,7 @@ async def a_copy_from_container(self):
             container=self.source_container
         )
         LOGGER.debug(f"Container url: {container_url}")
+        objects: typing.Union[typing.List, str]
         async with self.client.get(
                 common.generate_download_url(
                     self.source_host,

swift_upload_runner/server.py

@@ -70,14 +70,14 @@ async def servinit() -> aiohttp.web.Application:
 
 async def kill_client(
         app: aiohttp.web.Application
-):
+) -> None:
     """Kill the app client session."""
     await app["client"].close()
 
 
 def run_server(
         app: typing.Union[typing.Coroutine, aiohttp.web.Application]
-):
+) -> None:
     """Run the server."""
     aiohttp.web.run_app(
         app,
@@ -86,7 +86,7 @@ def run_server(
     )
 
 
-def main():
+def main() -> None:
     """."""
     run_server(servinit())