Merge pull request #135 from nexB/api-update

Api update

Author: JonoYang

matchcode-toolkit/CHANGELOG.rst

@@ -1,6 +1,16 @@
 Changelog
 =========
 
+v1.1.1
+------
+
+*2023-06-29* -- Do not include empty files when computing directory fingerprints.
+
+v1.1.0
+------
+
+*2023-06-22* -- Rename ``compute_directory_fingerprints`` to ``compute_codebase_directory_fingerprints`` and create a new version of ``compute_directory_fingerprints`` that works on Resource objects instead of codebases.
+
 v1.0.0
 ------
 

matchcode-toolkit/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matchcode-toolkit"
-version = "1.0.0"
+version = "1.1.1"
 
 [build-system]
 requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 6"]

matchcode-toolkit/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = matchcode-toolkit
-version = 1.0.0
+version = 1.1.1
 license = Apache-2.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

matchcode-toolkit/src/matchcode_toolkit/fingerprinting.py

@@ -69,30 +69,50 @@ def create_structure_fingerprint(directory, children):
     return _create_directory_fingerprint(features)
 
 
-def compute_directory_fingerprints(codebase):
+def _compute_directory_fingerprints(directory, codebase):
     """
-    Compute fingerprints for a directory from `codebase`
+    Compute fingerprints for `directory` from `codebase`
     """
-    for resource in codebase.walk(topdown=False):
-        if resource.is_file or not resource.path:
-            continue
-        children = [r for r in resource.walk(codebase) if r.is_file]
-        if len(children) == 1:
-            continue
+    # We do not want to add empty files to our fingerprint
+    children = [r for r in directory.walk(codebase) if r.is_file and r.size]
+    if len(children) == 1:
+        return
 
-        directory_content_fingerprint = create_content_fingerprint(children)
-        if hasattr(resource, 'directory_content_fingerprint'):
-            resource.directory_content_fingerprint = directory_content_fingerprint
-        else:
-            resource.extra_data['directory_content'] = directory_content_fingerprint
+    directory_content_fingerprint = create_content_fingerprint(children)
+    if hasattr(directory, 'directory_content_fingerprint'):
+        directory.directory_content_fingerprint = directory_content_fingerprint
+    else:
+        directory.extra_data['directory_content'] = directory_content_fingerprint
 
-        directory_structure_fingerprint = create_structure_fingerprint(resource, children)
-        if hasattr(resource, 'directory_structure_fingerprint'):
-            resource.directory_structure_fingerprint = directory_structure_fingerprint
-        else:
-            resource.extra_data['directory_structure'] = create_structure_fingerprint(resource, children)
+    directory_structure_fingerprint = create_structure_fingerprint(directory, children)
+    if hasattr(directory, 'directory_structure_fingerprint'):
+        directory.directory_structure_fingerprint = directory_structure_fingerprint
+    else:
+        directory.extra_data['directory_structure'] = directory_structure_fingerprint
+
+    directory.save(codebase)
+    return directory
+
+
+def compute_directory_fingerprints(directory, codebase):
+    """
+    Recursivly compute fingerprints for `directory` from `codebase`
+    """
+    for resource in directory.walk(codebase, topdown=False):
+        if resource.is_file:
+            continue
+        _ = _compute_directory_fingerprints(resource, codebase)
+    return directory
 
-        resource.save(codebase)
+
+def compute_codebase_directory_fingerprints(codebase):
+    """
+    Compute fingerprints for directories from `codebase`
+    """
+    for resource in codebase.walk(topdown=False):
+        if resource.is_file or not resource.path:
+            continue
+        _ = _compute_directory_fingerprints(resource, codebase)
     return codebase
 
 

matchcode-toolkit/src/matchcode_toolkit/pipelines/scan_and_fingerprint_package.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-from matchcode_toolkit.fingerprinting import compute_directory_fingerprints
+from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints
 
 from scanpipe.pipelines.scan_package import ScanPackage
 from scanpipe.pipes.codebase import ProjectCodebase
@@ -63,4 +63,4 @@ def fingerprint_codebase(self):
         Compute directory fingerprints for matching purposes
         """
         project_codebase = ProjectCodebase(self.project)
-        compute_directory_fingerprints(project_codebase)
+        compute_codebase_directory_fingerprints(project_codebase)

matchcode-toolkit/src/matchcode_toolkit/plugin_fingerprint.py

@@ -11,7 +11,7 @@
 
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import POST_SCAN_GROUP
-from matchcode_toolkit.fingerprinting import compute_directory_fingerprints
+from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints
 from plugincode.post_scan import post_scan_impl
 from plugincode.post_scan import PostScanPlugin
 
@@ -41,4 +41,4 @@ def is_enabled(self, fingerprint, **kwargs):
         return fingerprint
 
     def process_codebase(self, codebase, **kwargs):
-        codebase = compute_directory_fingerprints(codebase)
+        codebase = compute_codebase_directory_fingerprints(codebase)

matchcode-toolkit/tests/test_fingerprinting.py

@@ -14,7 +14,7 @@
 
 from matchcode_toolkit.fingerprinting import _create_directory_fingerprint
 from matchcode_toolkit.fingerprinting import _get_resource_subpath
-from matchcode_toolkit.fingerprinting import compute_directory_fingerprints
+from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints
 from matchcode_toolkit.fingerprinting import create_content_fingerprint
 from matchcode_toolkit.fingerprinting import create_halohash_chunks
 from matchcode_toolkit.fingerprinting import create_structure_fingerprint
@@ -95,10 +95,10 @@ def test_create_halohash_chunks(self):
         self.assertEqual(chunk3, expected_chunk3)
         self.assertEqual(chunk4, expected_chunk4)
 
-    def test_compute_directory_fingerprints(self):
+    def test_compute_codebase_directory_fingerprints(self):
         scan_loc = self.get_test_loc('abbrev-1.0.3-i.json')
         vc = VirtualCodebase(location=scan_loc)
-        vc = compute_directory_fingerprints(vc)
+        vc = compute_codebase_directory_fingerprints(vc)
         directory_content = vc.root.extra_data['directory_content']
         directory_structure = vc.root.extra_data['directory_structure']
         expected_directory_content = '0000000346ce04751a3c98f00086f16a91d9790b'

matchcode/api.py

@@ -129,6 +129,28 @@ class MultipleCharFilter(MultipleChoiceFilter):
     field_class = MultipleCharField
 
 
+# TODO: Think of a better name for this filter
+class MultipleCharInFilter(MultipleCharFilter):
+    def filter(self, qs, value):
+        if not value:
+            # Even though not a noop, no point filtering if empty.
+            return qs
+
+        if self.is_noop(qs, value):
+            return qs
+
+        predicate = self.get_filter_predicate(value)
+        old_field_name = next(iter(predicate))
+        new_field_name = f'{old_field_name}__in'
+        predicate[new_field_name] = predicate[old_field_name]
+        predicate.pop(old_field_name)
+
+        q = Q(**predicate)
+        qs = self.get_method(qs)(q)
+
+        return qs.distinct() if self.distinct else qs
+
+
 class MultipleSHA1Filter(MultipleCharFilter):
     """
     Overrides `MultipleCharFilter.filter()` to convert the SHA1

matchcode/indexing.py

@@ -13,7 +13,7 @@
 
 from commoncode.resource import VirtualCodebase
 
-from matchcode_toolkit.fingerprinting import compute_directory_fingerprints
+from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints
 from matchcode.models import ApproximateDirectoryContentIndex
 from matchcode.models import ApproximateDirectoryStructureIndex
 from matchcode.models import ExactPackageArchiveIndex
@@ -150,5 +150,5 @@ def index_package_directories(package):
     if not vc:
         return 0, 0
 
-    vc = compute_directory_fingerprints(vc)
+    vc = compute_codebase_directory_fingerprints(vc)
     return index_directory_fingerprints(vc, package)

matchcode/tests/test_index_packages.py

@@ -11,7 +11,7 @@
 
 from commoncode.resource import VirtualCodebase
 
-from matchcode_toolkit.fingerprinting import compute_directory_fingerprints
+from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints
 from matchcode_toolkit.fingerprinting import hexstring_to_binarray
 from matchcode.indexing import _create_virtual_codebase_from_package_resources
 from matchcode.indexing import index_directory_fingerprints
@@ -155,7 +155,7 @@ def test__create_virtual_codebase_from_package_resources(self):
 
     def test_index_directory_fingerprints(self):
         vc = _create_virtual_codebase_from_package_resources(self.test_package1)
-        vc = compute_directory_fingerprints(vc)
+        vc = compute_codebase_directory_fingerprints(vc)
 
         # Ensure tables are empty prior to indexing
         self.assertFalse(ApproximateDirectoryContentIndex.objects.all())