From 361653b392cf51053dd0d64c61073f05f319907e Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sat, 31 Oct 2020 13:43:01 -0700 Subject: [PATCH 1/6] cmdline: add to the test --- test-data/unit/cmdline.test | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 271b7c4f3e68..3654b58f21a4 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -810,15 +810,19 @@ def bar(a: int, b: int) -> str: src/anamespace/foo/bar.py:2: error: Incompatible return value type (got "int", expected "str") [case testNestedPEP420Packages] -# cmd: mypy -p bottles --namespace-packages -[file bottles/jars/secret/glitter.py] +# cmd: mypy -p pkg --namespace-packages +[file pkg/a1/b/c/d/e.py] x = 0 # type: str -[file bottles/jars/sprinkle.py] -from bottles.jars.secret.glitter import x -x + 1 +[file pkg/a1/b/f.py] +from pkg.a1.b.c.d.e import x +[file pkg/a2/__init__.py] +[file pkg/a2/b/c/d/e.py] +x = 0 # type: str +[file pkg/a2/b/f.py] +from pkg.a1.b.c.d.e import x [out] -bottles/jars/secret/glitter.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") -bottles/jars/sprinkle.py:2: error: Unsupported operand types for + ("str" and "int") +pkg/a1/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") +pkg/a2/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") [case testFollowImportStubs1] # cmd: mypy main.py From 35d5c69cfe44e3f13b1ebd789892da7bacb5a271 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sat, 31 Oct 2020 13:56:29 -0700 Subject: [PATCH 2/6] modulefinder: remove unused argument This was removed in #9616 --- mypy/main.py | 2 +- mypy/modulefinder.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 7de1f57dfece..763bd9e95638 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -930,7 +930,7 @@ def set_strict_flags() -> None: ()) targets = [] # TODO: use the same cache that the BuildManager will - cache = FindModuleCache(search_paths, fscache, options, special_opts.packages) + cache = FindModuleCache(search_paths, fscache, options) for p in special_opts.packages: if os.sep in p or os.altsep and os.altsep in p: fail("Package name '{}' cannot have a slash in it.".format(p), diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index e2fce6e46cfd..7675c69830a2 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -106,8 +106,7 @@ class FindModuleCache: def __init__(self, search_paths: SearchPaths, fscache: Optional[FileSystemCache] = None, - options: Optional[Options] = None, - ns_packages: Optional[List[str]] = None) -> None: + options: Optional[Options] = None) -> None: self.search_paths = search_paths self.fscache = fscache or FileSystemCache() # Cache for get_toplevel_possibilities: @@ -117,7 +116,6 @@ def __init__(self, self.results = {} # type: Dict[str, ModuleSearchResult] self.ns_ancestors = {} # type: Dict[str, str] self.options = options - self.ns_packages = ns_packages or [] # type: List[str] def clear(self) -> None: self.results.clear() From f1668e78942d6d4a3c4e9c92105b5a19d664f1e7 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sat, 31 Oct 2020 14:05:45 -0700 Subject: [PATCH 3/6] FindModuleCache: make arguments non optional This is important since the presence or absence of self.options.namespace_packages should be visible --- mypy/modulefinder.py | 6 +++--- mypy/stubgen.py | 2 +- mypy/stubtest.py | 4 +++- mypy/test/testcheck.py | 2 +- mypy/test/testmodulefinder.py | 8 ++++---- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 7675c69830a2..1dd5c56381f7 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -105,8 +105,8 @@ class FindModuleCache: def __init__(self, search_paths: SearchPaths, - fscache: Optional[FileSystemCache] = None, - options: Optional[Options] = None) -> None: + fscache: Optional[FileSystemCache], + options: Optional[Options]) -> None: self.search_paths = search_paths self.fscache = fscache or FileSystemCache() # Cache for get_toplevel_possibilities: @@ -206,7 +206,7 @@ def _can_find_module_in_parent_dir(self, id: str) -> bool: of the current working directory. """ working_dir = os.getcwd() - parent_search = FindModuleCache(SearchPaths((), (), (), ())) + parent_search = FindModuleCache(SearchPaths((), (), (), ()), self.fscache, self.options) while any(file.endswith(("__init__.py", "__init__.pyi")) for file in os.listdir(working_dir)): working_dir = os.path.dirname(working_dir) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 84b79715f5f8..0678ebc64ae3 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -1308,7 +1308,7 @@ def find_module_paths_using_search(modules: List[str], packages: List[str], result = [] # type: List[StubSource] typeshed_path = default_lib_path(mypy.build.default_data_dir(), pyversion, None) search_paths = SearchPaths(('.',) + tuple(search_path), (), (), tuple(typeshed_path)) - cache = FindModuleCache(search_paths) + cache = FindModuleCache(search_paths, fscache=None, options=None) for module in modules: m_result = cache.find_module(module) if isinstance(m_result, ModuleNotFoundReason): diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 79a79dac7cbc..8dd06afcf020 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -955,7 +955,9 @@ def build_stubs(modules: List[str], options: Options, find_submodules: bool = Fa """ data_dir = mypy.build.default_data_dir() search_path = mypy.modulefinder.compute_search_paths([], options, data_dir) - find_module_cache = mypy.modulefinder.FindModuleCache(search_path) + find_module_cache = mypy.modulefinder.FindModuleCache( + search_path, fscache=None, options=options + ) all_modules = [] sources = [] diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index f266a474a59a..44b93b90337b 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -341,7 +341,7 @@ def parse_module(self, module_names = m.group(1) out = [] search_paths = SearchPaths((test_temp_dir,), (), (), ()) - cache = FindModuleCache(search_paths) + cache = FindModuleCache(search_paths, fscache=None, options=None) for module_name in module_names.split(' '): path = cache.find_module(module_name) assert isinstance(path, str), "Can't find ad hoc case file: %s" % module_name diff --git a/mypy/test/testmodulefinder.py b/mypy/test/testmodulefinder.py index 4bed6720ac1c..4f839f641e7b 100644 --- a/mypy/test/testmodulefinder.py +++ b/mypy/test/testmodulefinder.py @@ -32,11 +32,11 @@ def setUp(self) -> None: ) options = Options() options.namespace_packages = True - self.fmc_ns = FindModuleCache(self.search_paths, options=options) + self.fmc_ns = FindModuleCache(self.search_paths, fscache=None, options=options) options = Options() options.namespace_packages = False - self.fmc_nons = FindModuleCache(self.search_paths, options=options) + self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options) def test__no_namespace_packages__nsx(self) -> None: """ @@ -159,11 +159,11 @@ def setUp(self) -> None: ) options = Options() options.namespace_packages = True - self.fmc_ns = FindModuleCache(self.search_paths, options=options) + self.fmc_ns = FindModuleCache(self.search_paths, fscache=None, options=options) options = Options() options.namespace_packages = False - self.fmc_nons = FindModuleCache(self.search_paths, options=options) + self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options) def path(self, *parts: str) -> str: return os.path.join(self.package_dir, *parts) From 750c92e282faaaa52c2426c847d5c05b0f1451e9 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sat, 31 Oct 2020 15:34:49 -0700 Subject: [PATCH 4/6] modulefinder: actually handle namespace packages correctly --- mypy/modulefinder.py | 67 +++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 1dd5c56381f7..f41354c5f974 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -362,36 +362,45 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: if isinstance(module_path, ModuleNotFoundReason): return [] result = [BuildSource(module_path, module, None)] + + package_path = None if module_path.endswith(('__init__.py', '__init__.pyi')): - # Subtle: this code prefers the .pyi over the .py if both - # exists, and also prefers packages over modules if both x/ - # and x.py* exist. How? We sort the directory items, so x - # comes before x.py and x.pyi. But the preference for .pyi - # over .py is encoded in find_module(); even though we see - # x.py before x.pyi, find_module() will find x.pyi first. We - # use hits to avoid adding it a second time when we see x.pyi. - # This also avoids both x.py and x.pyi when x/ was seen first. - hits = set() # type: Set[str] - for item in sorted(self.fscache.listdir(os.path.dirname(module_path))): - abs_path = os.path.join(os.path.dirname(module_path), item) - if os.path.isdir(abs_path) and \ - (os.path.isfile(os.path.join(abs_path, '__init__.py')) or - os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): - hits.add(item) - result += self.find_modules_recursive(module + '.' + item) - elif item != '__init__.py' and item != '__init__.pyi' and \ - item.endswith(('.py', '.pyi')): - mod = item.split('.')[0] - if mod not in hits: - hits.add(mod) - result += self.find_modules_recursive(module + '.' + mod) - elif os.path.isdir(module_path): - # Even subtler: handle recursive decent into PEP 420 - # namespace packages that are explicitly listed on the command - # line with -p/--packages. - for item in sorted(self.fscache.listdir(module_path)): - item, _ = os.path.splitext(item) - result += self.find_modules_recursive(module + '.' + item) + package_path = os.path.dirname(module_path) + elif self.fscache.isdir(module_path): + package_path = module_path + if package_path is None: + return result + + # This logic closely mirrors that in find_sources. One small but important difference is + # that we do not sort names with keyfunc. The recursive call to find_modules_recursive + # calls find_module, which will handle the preference between packages, pyi and py. + # Another difference is it doesn't handle nested search paths / package roots. + + seen = set() # type: Set[str] + names = self.fscache.listdir(package_path) + for name in names: + # Skip certain names altogether + if name == '__pycache__' or name.startswith('.') or name.endswith('~'): + continue + path = os.path.join(package_path, name) + + if self.fscache.isdir(path): + # Only recurse into packages + if (self.options and self.options.namespace_packages) or ( + self.fscache.isfile(os.path.join(path, "__init__.py")) + or self.fscache.isfile(os.path.join(path, "__init__.pyi")) + ): + seen.add(name) + result.extend(self.find_modules_recursive(module + '.' + name)) + else: + stem, suffix = os.path.splitext(name) + if stem == '__init__': + continue + if stem not in seen and '.' not in stem and suffix in PYTHON_EXTENSIONS: + # (If we sorted names) we could probably just make the BuildSource ourselves, + # but this ensures compatibility with find_module / the cache + seen.add(stem) + result.extend(self.find_modules_recursive(module + '.' + stem)) return result From df1487b376acb2e09dc6e85c8ebbc5fdf486371c Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sat, 31 Oct 2020 16:23:50 -0700 Subject: [PATCH 5/6] change import in test, also use the import Something in Python 3.5 orders things differently. This is what I meant anyway, so it's fine. --- test-data/unit/cmdline.test | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 3654b58f21a4..d76587cc8cd0 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -815,14 +815,19 @@ src/anamespace/foo/bar.py:2: error: Incompatible return value type (got "int", e x = 0 # type: str [file pkg/a1/b/f.py] from pkg.a1.b.c.d.e import x +x + 1 + [file pkg/a2/__init__.py] [file pkg/a2/b/c/d/e.py] x = 0 # type: str [file pkg/a2/b/f.py] -from pkg.a1.b.c.d.e import x +from pkg.a2.b.c.d.e import x +x + 1 [out] pkg/a1/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") pkg/a2/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") +pkg/a1/b/f.py:2: error: Unsupported operand types for + ("str" and "int") +pkg/a2/b/f.py:2: error: Unsupported operand types for + ("str" and "int") [case testFollowImportStubs1] # cmd: mypy main.py From 75ca3acba25b4ab57a514ff4bcc26e6527a790cc Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sat, 31 Oct 2020 18:00:16 -0700 Subject: [PATCH 6/6] sort to avoid py35 test issues --- mypy/modulefinder.py | 2 +- test-data/unit/cmdline.test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index f41354c5f974..d61c65e279bf 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -377,7 +377,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: # Another difference is it doesn't handle nested search paths / package roots. seen = set() # type: Set[str] - names = self.fscache.listdir(package_path) + names = sorted(self.fscache.listdir(package_path)) for name in names: # Skip certain names altogether if name == '__pycache__' or name.startswith('.') or name.endswith('~'): diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index d76587cc8cd0..28cad98a2283 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -824,10 +824,10 @@ x = 0 # type: str from pkg.a2.b.c.d.e import x x + 1 [out] -pkg/a1/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") pkg/a2/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") -pkg/a1/b/f.py:2: error: Unsupported operand types for + ("str" and "int") +pkg/a1/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") pkg/a2/b/f.py:2: error: Unsupported operand types for + ("str" and "int") +pkg/a1/b/f.py:2: error: Unsupported operand types for + ("str" and "int") [case testFollowImportStubs1] # cmd: mypy main.py