[Mypy] Add type check with mypy and fix storage (skypilot-org#1519)

* Add mypy for sky/data/storage.py

* not implemented function

* Fix format

* fix

* pyproject

* fix

* switch to python 3.8

* add requirements-dev.txt

* fix format

* ignore imports

* fix handle

* fix handle

* use mypy file

* Fix

* yapf

Author: Michaelvll

.github/workflows/mypy-generic.yml

@@ -0,0 +1,22 @@
+# This is needed for GitHub Actions for the "Waiting for status to be reported" problem,
+# according to https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks
+name: mypy
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - master
+      - 'releases/**'
+  pull_request:
+    branches:
+      - master
+      - 'releases/**'
+jobs:
+  mypy:
+    # Need to specify 20.04, because ubuntu-latest does not work with
+    # python 3.6: https://github.com/actions/setup-python/issues/355#issuecomment-1335042510
+    runs-on: ubuntu-20.04
+    steps:
+      - run: 'echo "No mypy to run"'

.github/workflows/mypy.yml

@@ -0,0 +1,34 @@
+name: mypy
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - master
+      - 'releases/**'
+  pull_request:
+    branches:
+      - master
+      - 'releases/**'
+jobs:
+  mypy:
+    # Need to specify 20.04, because ubuntu-latest does not work with
+    # python 3.6: https://github.com/actions/setup-python/issues/355#issuecomment-1335042510
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ["3.8"]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install mypy==$(grep mypy requirements-dev.txt | cut -d'=' -f3)
+    - name: Running mypy
+      run: |
+        mypy @tests/mypy_files.txt

format.sh

@@ -24,6 +24,7 @@ builtin cd "$ROOT" || exit 1
 YAPF_VERSION=$(yapf --version | awk '{print $2}')
 PYLINT_VERSION=$(pylint --version | head -n 1 | awk '{print $2}')
 PYLINT_QUOTES_VERSION=$(pip list | grep pylint-quotes | awk '{print $2}')
+MYPY_VERSION=$(mypy --version | awk '{print $2}')
 
 # # params: tool name, tool version, required version
 tool_version_check() {
@@ -33,9 +34,10 @@ tool_version_check() {
     fi
 }
 
-tool_version_check "yapf" $YAPF_VERSION "0.32.0"
-tool_version_check "pylint" $PYLINT_VERSION "2.8.2"
-tool_version_check "pylint-quotes" $PYLINT_QUOTES_VERSION "0.2.3"
+tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-dev.txt | cut -d'=' -f3)"
+tool_version_check "pylint" $PYLINT_VERSION "$(grep "pylint==" requirements-dev.txt | cut -d'=' -f3)"
+tool_version_check "pylint-quotes" $PYLINT_QUOTES_VERSION "$(grep "pylint-quotes==" requirements-dev.txt | cut -d'=' -f3)"
+tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)"
 
 YAPF_FLAGS=(
     '--recursive'
@@ -87,6 +89,12 @@ else
    format_changed
 fi
 
+# Run mypy
+# TODO(zhwu): When more of the codebase is typed properly, the mypy flags
+# should be set to do a more stringent check.
+echo 'SkyPilot mypy:'
+mypy @tests/mypy_files.txt
+
 # Run Pylint
 echo 'Sky Pylint:'
 pylint --load-plugins pylint_quotes sky

pyproject.toml

@@ -16,3 +16,8 @@ env = [
     "SKYPILOT_DEBUG=1",
     "SKYPILOT_DISABLE_USAGE_COLLECTION=1"
 ]
+
+[tool.mypy]
+python_version = "3.8"
+follow_imports = "skip"
+ignore_missing_imports = true

requirements-dev.txt

@@ -5,6 +5,9 @@ pylint==2.8.2
 pylint-quotes==0.2.3
 toml==0.10.2
 
+# type checking
+mypy==0.991
+
 # testing
 pytest
 pytest-xdist

sky/adaptors/azure.py

@@ -13,7 +13,7 @@ def wrapper(*args, **kwargs):
         global azure
         if azure is None:
             try:
-                import azure as _azure
+                import azure as _azure  # type: ignore
                 azure = _azure
             except ImportError:
                 raise ImportError('Fail to import dependencies for Azure.'

sky/data/data_utils.py

@@ -3,7 +3,6 @@
 from multiprocessing import pool
 import os
 import subprocess
-from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Tuple
 import urllib.parse
 
@@ -110,7 +109,7 @@ def _group_files_by_dir(
     return grouped_files, dirs
 
 
-def parallel_upload(source_path_list: List[Path],
+def parallel_upload(source_path_list: List[str],
                     filesync_command_generator: Callable[[str, List[str]], str],
                     dirsync_command_generator: Callable[[str, str], str],
                     bucket_name: str,