Setup a minimal mypy config (#1268)

* Minimal mypy setup

* Fix mypy errors

* Use `from __future__ import annotations`

Author: alanhdu

.github/workflows/main.yaml

@@ -68,6 +68,22 @@ jobs:
       - uses: actions/setup-python@main
       - uses: pre-commit/action@main
 
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup conda
+        uses: mamba-org/provision-with-micromamba@main
+        with:
+          environment-file: ci/environment-typecheck.yml
+
+      - name: mypy
+        shell: bash -l {0}
+        run: |
+          mypy fsspec
+
   downstream:
     name: downstream
     runs-on: ubuntu-latest

ci/environment-typecheck.yml

@@ -0,0 +1,13 @@
+name: test_env
+channels:
+  - conda-forge
+dependencies:
+  - mypy=1.3
+  - pyarrow
+  - python=3.8
+  - pip
+  - pip:
+    - types-paramiko
+    - types-requests
+    - types-tqdm
+    - types-ujson

fsspec/_version.py

@@ -51,7 +51,6 @@ class NotThisMethod(Exception):
     """Exception raised if a method is not valid for the current scenario."""
 
 
-LONG_VERSION_PY = {}
 HANDLERS = {}
 
 

fsspec/asyn.py

@@ -9,7 +9,7 @@
 import threading
 from contextlib import contextmanager
 from glob import has_magic
-from typing import Iterable
+from typing import TYPE_CHECKING, Iterable
 
 from .callbacks import _DEFAULT_CALLBACK
 from .exceptions import FSTimeoutError
@@ -154,13 +154,18 @@ def get_loop():
     return loop[0]
 
 
-try:
+if TYPE_CHECKING:
     import resource
-except ImportError:
-    resource = None
-    ResourceError = OSError
+
+    ResourceError = resource.error
 else:
-    ResourceError = getattr(resource, "error", IOError)
+    try:
+        import resource
+    except ImportError:
+        resource = None
+        ResourceError = OSError
+    else:
+        ResourceError = getattr(resource, "error", IOError)
 
 _DEFAULT_BATCH_SIZE = 128
 _NOFILES_DEFAULT_BATCH_SIZE = 1280

fsspec/config.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 import configparser
 import json
 import os
 import warnings
+from typing import Any
 
-conf = {}
+conf: dict[str, dict[str, Any]] = {}
 default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/fsspec")
 conf_dir = os.environ.get("FSSPEC_CONFIG_DIR", default_conf_dir)
 

fsspec/gui.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import re
+from typing import ClassVar, Sequence
 
 import panel as pn
 
@@ -25,9 +26,11 @@ class SigSlot(object):
     By default, all signals emit a DEBUG logging statement.
     """
 
-    signals = []  # names of signals that this class may emit
-    # each of which must be set by _register for any new instance
-    slots = []  # names of actions that this class may respond to
+    # names of signals that this class may emit each of which must be
+    # set by _register for any new instance
+    signals: ClassVar[Sequence[str]] = []
+    # names of actions that this class may respond to
+    slots: ClassVar[Sequence[str]] = []
 
     # each of which must be a method name
 

fsspec/implementations/cached.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import contextlib
 import hashlib
 import inspect
@@ -7,6 +9,7 @@
 import tempfile
 import time
 from shutil import rmtree
+from typing import ClassVar
 
 from fsspec import AbstractFileSystem, filesystem
 from fsspec.callbacks import _DEFAULT_CALLBACK
@@ -39,7 +42,7 @@ class CachingFileSystem(AbstractFileSystem):
       allowed, for testing
     """
 
-    protocol = ("blockcache", "cached")
+    protocol: ClassVar[str | tuple[str, ...]] = ("blockcache", "cached")
 
     def __init__(
         self,

fsspec/implementations/memory.py

@@ -1,9 +1,10 @@
-from __future__ import absolute_import, division, print_function
+from __future__ import absolute_import, annotations, division, print_function
 
 import logging
 from datetime import datetime
 from errno import ENOTEMPTY
 from io import BytesIO
+from typing import Any, ClassVar
 
 from fsspec import AbstractFileSystem
 
@@ -17,7 +18,7 @@ class MemoryFileSystem(AbstractFileSystem):
     in memory filesystem.
     """
 
-    store = {}  # global, do not overwrite!
+    store: ClassVar[dict[str, Any]] = {}  # global, do not overwrite!
     pseudo_dirs = [""]  # global, do not overwrite!
     protocol = "memory"
     root_marker = "/"

fsspec/implementations/reference.py

@@ -6,13 +6,15 @@
 import math
 import os
 from functools import lru_cache
+from typing import TYPE_CHECKING
 
 import fsspec.core
 
 try:
     import ujson as json
 except ImportError:
-    import json
+    if not TYPE_CHECKING:
+        import json
 
 from ..asyn import AsyncFileSystem
 from ..callbacks import _DEFAULT_CALLBACK
@@ -803,11 +805,11 @@ def cat(self, path, recursive=False, on_error="raise", **kwargs):
                     urls.append(u)
                     starts.append(s)
                     ends.append(e)
-                except FileNotFoundError as e:
+                except FileNotFoundError as err:
                     if on_error == "raise":
                         raise
                     if on_error != "omit":
-                        out[p] = e
+                        out[p] = err
 
             # process references into form for merging
             urls2 = []
@@ -924,7 +926,6 @@ def _render_jinja(u):
         self.references.update(self._process_gen(references.get("gen", [])))
 
     def _process_templates(self, tmp):
-
         self.templates = {}
         if self.template_overrides is not None:
             tmp.update(self.template_overrides)
@@ -939,7 +940,6 @@ def _process_templates(self, tmp):
                 self.templates[k] = v
 
     def _process_gen(self, gens):
-
         out = {}
         for gen in gens:
             dimension = {

fsspec/implementations/tar.py

@@ -81,7 +81,7 @@ def __init__(
 
         self._fo_ref = fo
         self.fo = fo  # the whole instance is a context
-        self.tar: tarfile.TarFile = tarfile.TarFile(fileobj=self.fo)
+        self.tar = tarfile.TarFile(fileobj=self.fo)
         self.dir_cache = None
 
         self.index_store = index_store

fsspec/implementations/tests/test_tar.py

@@ -1,10 +1,11 @@
+from __future__ import annotations
+
 import os
 import shutil
 import tarfile
 import tempfile
 from io import BytesIO
 from pathlib import Path
-from typing import Dict
 
 import pytest
 
@@ -210,7 +211,7 @@ def test_ls_with_folders(compression: str, tmp_path: Path):
     but make sure that the reading filesystem is still able to resolve the
     intermediate folders, like the ZipFileSystem.
     """
-    tar_data: Dict[str, bytes] = {
+    tar_data: dict[str, bytes] = {
         "a.pdf": b"Hello A!",
         "b/c.pdf": b"Hello C!",
         "d/e/f.pdf": b"Hello F!",

fsspec/registry.py

@@ -1,11 +1,13 @@
+from __future__ import annotations
+
 import importlib
 import types
 import warnings
 
 __all__ = ["registry", "get_filesystem_class", "default"]
 
 # internal, mutable
-_registry = {}
+_registry: dict[str, type] = {}
 
 # external, immutable
 registry = types.MappingProxyType(_registry)

fsspec/spec.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import io
 import logging
 import os
@@ -7,6 +9,7 @@
 from errno import ESPIPE
 from glob import has_magic
 from hashlib import sha256
+from typing import ClassVar
 
 from .callbacks import _DEFAULT_CALLBACK
 from .config import apply_config, conf
@@ -101,7 +104,7 @@ class AbstractFileSystem(metaclass=_Cached):
     _cached = False
     blocksize = 2**22
     sep = "/"
-    protocol = "abstract"
+    protocol: ClassVar[str | tuple[str, ...]] = "abstract"
     _latest = None
     async_impl = False
     mirror_sync_methods = False

fsspec/utils.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 import math
 import os
@@ -110,7 +112,7 @@ def update_storage_options(options, inherited=None):
 
 
 # Compression extensions registered via fsspec.compression.register_compression
-compressions = {}
+compressions: dict[str, str] = {}
 
 
 def infer_compression(filename):

setup.cfg

@@ -37,3 +37,18 @@ skip=
     docs/source/conf.py
     versioneer.py
     fsspec/_version
+
+[mypy]
+follow_imports = normal
+ignore_missing_imports = True
+enable_error_code = ignore-without-code,truthy-bool,truthy-iterable,unused-awaitable
+
+disallow_untyped_decorators = True
+strict_equality = True
+warn_redundant_casts = True
+warn_unused_configs = True
+warn_unused_ignores = True
+
+
+# don't bother type-checking test_*.py or conftest.py files
+exclude = (test_.*|conftest)\.py$