Merge development branch 'kotopes-develop'

Author: Daniel Zagaynov

Diff for: README.md

@@ -155,6 +155,33 @@ But it is possible to ignore all deps, that are hidden inside contexts:
 %
 ```
 
+#### Matching dependencies with environment
+Imagine you write your big project, all your dependencies (including building and testing dependencies) are installed to your virtual (or real) environment. So you need to detect your runnning dependencies and match them to packages, installed to your environment, and get **requirements.txt** file, which you can include in your package. For such cases there is **--inspect_env** option:
+
+```shell
+% py3req --inspect_env --verbose src
+% cat requirements.txt
+numpy==2.2.1
+```
+
+As you can see, **py3req** saves matched dependencies to **requirements.txt** file.
+
+Now we can get your testing dependencies:
+```shell
+% py3req --inspect_env --verbose tests
+py3prov:INFO: bad name for provides from path:config-3.12-x86_64-linux-gnu
+py3prov:INFO: bad name for provides from path:numpy.libs
+py3req:/tmp/project/tests/test1.py: "unittest" lines:[1] is possibly a self-providing dependency, skip it
+The following deps:pytest was satisfied by package:pytest==8.3.4
+% cat requirements.txt
+pytest==8.3.4
+```
+
+The difference between running **py3req** with option **--inspect_env** and **pip3 freeze** is that the last command lists all packages installed to your environment (including their dependencies). But **py3req** just finds all dependencies of given sources and can match it to the installed packages.
+
+Also there is an extra option for **--inspect_env** which is called **--env_path**. This options lets you to specify path to your environment (where your packages are installed). It is usefull for **CI** or something like that, but by default **py3req** checks your [purelib](https://docs.python.org/3/library/sysconfig.html#installation-paths) and [platlib](https://docs.python.org/3/library/sysconfig.html#installation-paths), so you can skip this option.
+
+
 Other options are little bit specific, but there is clear **--help** option output. Please, check it.
 
 

Diff for: pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "py3dephell"
-version = "0.3.1"
+version = "0.4.1"
 authors = [
 {name = "Daniel Zagaynov", email = "[email protected]"},
 ]

Diff for: src/py3dephell/py3prov.py

@@ -3,6 +3,7 @@
 import os
 import re
 import sys
+import csv
 import argparse
 import sysconfig
 from pathlib import Path
@@ -37,7 +38,7 @@ def processing_pth(path):
             new_names = [os.path.join(path, new_dir) for new_dir in new_names]
             return new_names
     except FileNotFoundError:
-        print(f'py3prov: No such file or directory:{path}', file=sys.stderr)
+        print(f'py3prov:INFO: No such file or directory:{path}', file=sys.stderr)
         return []
 
 
@@ -79,7 +80,7 @@ def create_provides_from_path(path, prefixes=sys.path, abs_mode=False,
             path = Path(path.as_posix().replace(pref + '/', ''))
 
     if not path:
-        raise ValueError('py3prov.create_provides_from_path: path cannot be empty (possibly it was cut by pref)')
+        raise ValueError('py3prov.create_provides_from_path: path cannot be empty (possibly it was cut by prefix)')
 
     top_package_flag = False
 
@@ -101,7 +102,7 @@ def create_provides_from_path(path, prefixes=sys.path, abs_mode=False,
 
         if '.' in parts[-1]:
             if parts[-1] not in _bad_provides and verbose:
-                print(f'py3prov: bad name for provides from path:{path.as_posix()}', file=sys.stderr)
+                print(f'py3prov:INFO: bad name for provides from path:{path}', file=sys.stderr)
                 _bad_provides.add(parts[-1])
 
         if abs_mode and (all([part.isidentifier() for part in parts]) or not skip_wrong_names):
@@ -181,7 +182,7 @@ def module_detector(path, prefixes, modules=[], verbose_mode=True):
         if pref and (pref := os.path.normpath(pref)) and path.startswith(pref + '/') and pref != os.path.normpath(path):
             module = re.match(r'%s\/([^\/]+)' % re.escape(pref), path).groups()[0]
             if verbose_mode and module not in modules:
-                print(f'py3prov: detected potential module:{module}', file=sys.stderr)
+                print(f'py3prov:INFO: detected potential module:{module}', file=sys.stderr)
             return pref, module
     return None, None
 
@@ -202,20 +203,20 @@ def pth_detector(pathes, verbose_mode=False):
         path = Path(path)
         if not path.exists():
             if verbose_mode:
-                print(f'Path {path} does not exist, skip it', file=sys.stderr)
+                print(f'py3prov:WARNING: Path {path} does not exist, skip it', file=sys.stderr)
             continue
         if path.suffix == '.pth':
             if verbose_mode:
-                print(f'Detected .pth file:{path.absolute().as_posix()}', file=sys.stderr)
+                print(f'py3prov:INFO: Detected .pth file:{path.absolute().as_posix()}', file=sys.stderr)
             new_prefixes += processing_pth(path.absolute().as_posix())
         elif path.is_dir():
             for item in path.iterdir():
                 if item.suffix == '.pth':
-                    print(f'Detected .pth file:{item.absolute().as_posix()}', file=sys.stderr)
+                    print(f'py3prov:INFO: Detected .pth file:{item.absolute().as_posix()}', file=sys.stderr)
                     new_prefixes += processing_pth(item.absolute().as_posix())
         else:
             if verbose_mode:
-                print(f'Path {path} is not a directory or .pth file, skip it', file=sys.stderr)
+                print(f'py3prov:INFO: Path {path} is not a directory or .pth file, skip it', file=sys.stderr)
     return new_prefixes
 
 
@@ -265,6 +266,59 @@ def files_filter(files, prefixes=sys.path, only_prefix=False,
     return files_dict
 
 
+def _process_rec_file(file, verbose=False):
+    try:
+        with open(file) as f:
+            return list(filter(lambda p: suff in [so_suffix, shlib_suffix, soabi, soabi3, '.py', abi3]
+                               if (suff := Path(p).suffix) is not None else True,
+                               map(lambda row: row[0], csv.reader(f))))
+    except (FileNotFoundError, PermissionError) as err:
+        if verbose:
+            print(f"py3prov:WARNING: Failed to proceed {file} due to {err}", file=sys.stderr)
+
+
+def _genprov_from_recs(record, verbose=False):
+    if (recs := _process_rec_file(record, verbose=verbose)) is not None:
+        return sum(map(lambda path: create_provides_from_path(path, abs_mode=True,
+                                                              skip_namespace_pkgs=False, verbose=verbose),
+                       recs), start=[])
+
+
+def genprov_from_env(paths=[], verbose=False):
+    """
+    Generate provides from installed to environment wheels according to their .dist-info/RECORD file
+
+    :param paths: paths where py3prov should look for wheels.
+    If not set, wheels will be searched according to purelib and platlib
+    :type paths: list()
+    :param verbose: make it verbose
+    :type verbose: Bool
+    :return: dictionary from package name, package version and its provides
+    :rtype: dict
+    """
+    paths = set([sysconfig.get_paths()['purelib'], sysconfig.get_paths()['platlib']]) if paths == [] else paths
+    pattern = re.compile("([^/]+)-([^-]+)\.dist-info")
+    pkg_ver_provs = {}
+    for dist_inf, recs in _find_dist_info_recs(paths, verbose):
+        if (fnd := pattern.search(dist_inf.name)) is not None:
+            pkg, ver = fnd.groups()
+            provs = set(_genprov_from_recs(recs, verbose=verbose))
+            pkg_ver_provs[(pkg, ver)] = provs
+    return pkg_ver_provs
+
+
+def _find_dist_info_recs(paths, verbose=False):
+    for direc in paths:
+        for pkg in Path(direc).iterdir():
+            if (dist_inf := Path(pkg)).is_dir() and dist_inf.name.endswith(".dist-info"):
+                if (rec := dist_inf.joinpath("RECORD")).exists():
+                    yield dist_inf, rec
+                elif verbose:
+                    print("py3prov:WARNING: Found dist-info, which does not"
+                          f" provide RECORD file:{dist_inf.absolute().as_posix()}",
+                          file=sys.stderr)
+
+
 def generate_provides(files, prefixes=sys.path, skip_pth=False, only_prefix=False,
                       deep_search=False, abs_mode=False, verbose=True,
                       skip_wrong_names=True, skip_namespace_pkgs=True):

Diff for: src/py3dephell/py3req.py

@@ -9,7 +9,7 @@
 import argparse
 import pathlib
 import sysconfig
-from .py3prov import generate_provides, search_for_provides
+from .py3prov import generate_provides, search_for_provides, genprov_from_env
 
 
 def _remove_reduntant_args(func):
@@ -296,6 +296,7 @@ def filter_requirements(file, deps, provides=[], only_top_module=[], ignore_list
     :rtype: set[str]
     '''
     dependencies = set()
+
     for dep, lines in deps.items():
         if dep in ignore_list:
             if verbose:
@@ -311,14 +312,15 @@ def filter_requirements(file, deps, provides=[], only_top_module=[], ignore_list
             if only_top_module:
                 dep = dep.split('.')[0]
             dependencies.add(dep)
+
     return dependencies
 
 
 @_remove_reduntant_args
 def generate_requirements(files, add_prov_path=[], prefixes=sys.path,
                           ignore_list=sys.builtin_module_names, read_prov_from_file=None,
                           skip_subs=True, only_external_deps=False, only_top_module=False,
-                          exclude_stdlib=False, stderr=sys.stderr, verbose=True):
+                          exclude_stdlib=False, inspect_env=False, env_path=[], stderr=sys.stderr, verbose=True):
     '''
     Generate dependencies for given file-list, filter them through detected provides and return in specified format.
 
@@ -340,6 +342,10 @@ def generate_requirements(files, add_prov_path=[], prefixes=sys.path,
     :type only_top_module: Bool
     :param exclude_stdlib: exclude from dependencies standard lib provides
     :type exclude_stdlib: Bool
+    :param inspect_env: inspect environment for installed packages and match with them requirements
+    :type inspect_env: Bool
+    :param env_path: path to the environment (useful for inspect_env option)
+    :type env_path: [str]
     :param stderr: messages output
     :type stderr: io
     :param verbose: verbose flag
@@ -351,7 +357,11 @@ def generate_requirements(files, add_prov_path=[], prefixes=sys.path,
     abs_provides = set()
     add_provides = set()
     modules = {}
-    dependencies = {}
+    if not inspect_env:
+        dependencies = {}
+    else:
+        dependencies = set()
+        tmp_dependencies = set()
 
     if read_prov_from_file:
         with open(read_prov_from_file) as f:
@@ -383,10 +393,16 @@ def generate_requirements(files, add_prov_path=[], prefixes=sys.path,
         # This module is provided by different real modules which are platform specific, such as posixpath.py
         add_provides.add("os.path")
 
+    if inspect_env:
+        env_path = set([sysconfig.get_paths()['purelib'],
+                        sysconfig.get_paths()['platlib']]) if env_path == [] else env_path
+        env_provides = genprov_from_env(paths=env_path, verbose=verbose)
+
     for file in files:
         if file.endswith('.so'):
             if (dep := catch_so(file, stderr)):
-                dependencies[file] = set(), set(), set(), set([dep])
+                if not inspect_env:
+                    dependencies[file] = set(), set(), set(), set([dep])
                 continue
 
         abs_deps, rel_deps, adv_deps, skip =\
@@ -412,7 +428,26 @@ def generate_requirements(files, add_prov_path=[], prefixes=sys.path,
 
         filter_requirements(file, skip, skip_flag=True, stderr=stderr, verbose=verbose)
 
-        dependencies[file] = abs_deps, rel_deps, adv_deps, set()
+        if not inspect_env:
+            dependencies[file] = abs_deps, rel_deps, adv_deps, set()
+        else:
+            tmp_dependencies |= abs_deps | rel_deps | adv_deps
+
+    if inspect_env:
+        for pkg_ver, provs in env_provides.items():
+            if (matched := provs.intersection(tmp_dependencies)):
+                tmp_dependencies.difference_update(matched)
+                dependencies.add("==".join(pkg_ver))
+                if verbose:
+                    print(f"The following deps:{",".join(matched)} was satisfied by package:{"==".join(pkg_ver)}",
+                          file=sys.stderr)
+                if not tmp_dependencies:
+                    break
+        else:
+            if verbose and tmp_dependencies:
+                print("WARNING! Dependencies not matched to any package"
+                      f" in your environment:{",".join(tmp_dependencies)}",
+                      file=sys.stderr)
 
     return dependencies
 
@@ -423,7 +458,7 @@ def main():
     args.add_argument('--add_prov_path', default="",
                       help='List of additional paths for provides (separated by ":")')
     args.add_argument('--prefixes',
-                      help='Prefixes that will be removed from full'
+                      help='Prefixes that will be removed from fully '
                       'qualified name for relative import (string separated by ":")')
     args.add_argument('--ignore_list', default="",
                       help='List of dependencies that should be ignored (separated by ":")')
@@ -438,6 +473,12 @@ def main():
                       help='For dependency like a.b skip b')
     args.add_argument('--include_stdlib', action='store_true',
                       help='Exclude dependencies that are provided by installed python3 standart library')
+    args.add_argument("--inspect_env", action="store_true",
+                      help="Inspect environment for installed packages and "
+                      + "match required symbols to installed packages")
+    args.add_argument("--env_path", default="",
+                      help='Set path to the environment with installed packages (string separated by ":"). '
+                      + "By default set to purelib and platlib")
     args.add_argument('--verbose', action='store_true',
                       help='Verbose stderr')
     args.add_argument('input', nargs='*',
@@ -458,20 +499,26 @@ def main():
     if not args.include_built_in:
         ignore_list += sys.builtin_module_names
 
+    env_path = [p for p in args.env_path.split(":") if p]
+
     dependencies = generate_requirements(files=args.input, add_prov_path=args.add_prov_path.split(":"),
                                          ignore_list=ignore_list,
                                          read_prov_from_file=args.read_prov_from_file,
                                          skip_subs=True, prefixes=prefixes,
                                          only_external_deps=args.exclude_hidden_deps,
                                          only_top_module=args.only_top_module,
                                          exclude_stdlib=not args.include_stdlib,
+                                         inspect_env=args.inspect_env, env_path=env_path,
                                          verbose=args.verbose)
 
-    for file, deps in dependencies.items():
-        if any(deps) and args.verbose:
-            print(f'{file}:{" ".join([" ".join(req) for req in deps if req])}')
-        elif any(deps):
-            print('\n'.join(['\n'.join(req) for req in deps if req]))
+    if not args.inspect_env:
+        for file, deps in dependencies.items():
+            if any(deps) and args.verbose:
+                print(f'{file}:{" ".join([" ".join(req) for req in deps if req])}')
+            elif any(deps):
+                print('\n'.join(['\n'.join(req) for req in deps if req]))
+    else:
+        print("\n".join(dependencies))
 
 
 if __name__ == '__main__':

Diff for: tests/package.py

@@ -1,4 +1,7 @@
+import csv
+import hashlib
 import pathlib
+from shutil import rmtree
 
 
 def prepare_package(path, name, namespace_pkg=False, w_pth=False, level=0):
@@ -47,3 +50,31 @@ def generate_pymodule(path, name, text=None):
     p = pathlib.Path(path).joinpath(f'{name}.py')
     p.write_text(text)
     return [p] + generate_somodule(path, f'{name}_lib')
+
+
+def generate_install_wheel(path, name, version, broken=False):
+    """
+    Generate and install dummy wheel
+    """
+    p = pathlib.Path(path).joinpath(name)
+    try:
+        p.mkdir(parents=True)
+    except FileExistsError:
+        rmtree(p)
+        p.mkdir(parents=True)
+
+    for module, function in zip(["module_1.py", "module_2.py", "__init__.py"], ["func_1", "func_2", "func"]):
+        p.joinpath(module).write_text(f"def {function}:\n\tpass")
+
+    d_io = pathlib.Path(path).joinpath(f"{name}-{version}.dist-info")
+    try:
+        d_io.mkdir(parents=True)
+    except FileExistsError:
+        rmtree(d_io)
+        d_io.mkdir(parents=True)
+    if not broken:
+        with open(d_io.joinpath("RECORD"), "w", newline="") as csv_file:
+            csv_writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+            for module, function in zip(["module_1.py", "module_2.py", "__init__.py"], ["func_1", "func_2", "func"]):
+                text = f"def {function}:\n\tpass"
+                csv_writer.writerow([f"{name}/{module}", hashlib.sha256(text.encode()).hexdigest(), 666])