Merge branch 'main' into minecode-pipeline-npm

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

minecode_pipelines/__init__.py

@@ -8,4 +8,4 @@
 #
 
 
-VERSION = "0.0.1b17"
+VERSION = "0.0.1b23"

minecode_pipelines/miners/composer.py

@@ -0,0 +1,103 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+from minecode_pipelines.utils import get_temp_file
+import requests
+from packageurl import PackageURL
+
+
+def get_composer_packages():
+    """
+    Fetch all Composer packages from Packagist and save them to a temporary JSON file.
+    Response example:
+    {
+        "packageNames" ["0.0.0/composer-include-files", "0.0.0/laravel-env-shim"]
+    }
+    """
+
+    response = requests.get("https://packagist.org/packages/list.json")
+    if not response.ok:
+        return
+
+    packages = response.json()
+    temp_file = get_temp_file("ComposerPackages", "json")
+    with open(temp_file, "w", encoding="utf-8") as f:
+        json.dump(packages, f, indent=4)
+
+    return temp_file
+
+
+def get_composer_purl(vendor, package):
+    """
+    Fetch all available Package URLs (purls) for a Composer package from Packagist.
+    Response example:
+    {
+      "minified": "composer/2.0",
+      "packages": [
+        {
+          "monolog/monolog": {
+            "0": {
+              "name": "monolog/monolog",
+              "version": "3.9.0"
+            }
+          }
+        }
+      ],
+      "security-advisories": [
+        {
+          "advisoryId": "PKSA-dmw8-jd8k-q3c6",
+          "affectedVersions": ">=1.8.0,<1.12.0"
+        }
+      ]
+    }
+    get_composer_purl("monolog", "monolog")
+    -> ["pkg:composer/monolog/[email protected]", "pkg:composer/monolog/[email protected]", ...]
+    """
+    purls = []
+    url = f"https://repo.packagist.org/p2/{vendor}/{package}.json"
+
+    try:
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
+    except requests.RequestException:
+        return purls
+
+    data = response.json()
+    packages = data.get("packages", {})
+    releases = packages.get(f"{vendor}/{package}", [])
+
+    for release in releases:
+        version = release.get("version")
+        if version:
+            purl = PackageURL(
+                type="composer",
+                namespace=vendor,
+                name=package,
+                version=version,
+            )
+            purls.append(purl.to_string())
+
+    return purls
+
+
+def load_composer_packages(packages_file):
+    """Load and return a list of (vendor, package) tuples from a JSON file."""
+    with open(packages_file, encoding="utf-8") as f:
+        packages_data = json.load(f)
+
+    package_names = packages_data.get("packageNames", [])
+    result = []
+
+    for item in package_names:
+        if "/" in item:
+            vendor, package = item.split("/", 1)
+            result.append((vendor, package))
+
+    return result

minecode_pipelines/miners/cran.py

@@ -0,0 +1,66 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+from pathlib import Path
+import requests
+from packageurl import PackageURL
+
+
+def fetch_cran_db(output_file="cran_db.json") -> Path:
+    """
+    Download the CRAN package database (~250MB JSON) in a memory-efficient way.
+    Saves it to a file instead of loading everything into memory.
+    """
+
+    url = "https://crandb.r-pkg.org/-/all"
+    output_path = Path(output_file)
+
+    with requests.get(url, stream=True) as response:
+        response.raise_for_status()
+        with output_path.open("wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+    return output_path
+
+
+def extract_cran_packages(json_file_path: str) -> list:
+    """
+    Extract package names and their versions from a CRAN DB JSON file.
+    ex:
+    {
+      "AATtools": {
+      "_id": "AATtools",
+      "_rev": "8-9ebb721d05b946f2b437b49e892c9e8c",
+      "name": "AATtools",
+      "versions": {
+         "0.0.1": {...},
+         "0.0.2": {...},
+         "0.0.3": {...}
+      }
+    }
+    """
+    db_path = Path(json_file_path)
+    if not db_path.exists():
+        raise FileNotFoundError(f"File not found: {db_path}")
+
+    with open(db_path, encoding="utf-8") as f:
+        data = json.load(f)
+
+    for pkg_name, pkg_data in data.items():
+        versions = list(pkg_data.get("versions", {}).keys())
+        purls = []
+        for version in versions:
+            purl = PackageURL(
+                type="cran",
+                name=pkg_name,
+                version=version,
+            )
+            purls.append(purl.to_string())
+        yield purls

minecode_pipelines/miners/swift.py

@@ -0,0 +1,96 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import shutil
+import subprocess
+from urllib.parse import urlparse
+
+"""
+Clone the Swift Index repo (https://github.com/SwiftPackageIndex/PackageList) and the Minecode Pipelines Swift repo.
+Read the packages.json file from the Swift Index repo to get a list of Git repositories.
+Fetch the tags for each repo using the git ls-remote command,
+then create package URLs for each repo with its version and store them in the Minecode Pipelines Swift repo.
+"""
+
+
+def is_safe_repo_url(repo_url: str) -> bool:
+    """Return True if the URL is HTTPS GitHub with .git suffix or has at least two path segments."""
+    parsed = urlparse(repo_url)
+    return (
+        parsed.scheme == "https"
+        and parsed.netloc == "github.com"
+        and parsed.path.endswith(".git")
+        or parsed.path.count("/") >= 2
+    )
+
+
+def fetch_git_tags_raw(repo_url: str, timeout: int = 60, logger=None) -> str | None:
+    """Run `git ls-remote` on a GitHub repo and return raw output, or None on error."""
+    git_executable = shutil.which("git")
+    if git_executable is None:
+        logger("Git executable not found in PATH")
+        return None
+
+    if not is_safe_repo_url(repo_url):
+        raise ValueError(f"Unsafe repo URL: {repo_url}")
+
+    try:
+        result = subprocess.run(
+            [git_executable, "ls-remote", repo_url],
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=timeout,
+        )
+        return result.stdout.strip()
+    except subprocess.CalledProcessError as e:
+        logger(f"Failed to fetch tags for {repo_url}: {e}")
+    except subprocess.TimeoutExpired:
+        logger(f"Timeout fetching tags for {repo_url}")
+    return None
+
+
+# FIXME duplicated with miners github
+def split_org_repo(url_like):
+    """
+    Given a URL-like string to a GitHub repo or a repo name as in org/name,
+    split and return the org and name.
+
+    For example:
+    >>> split_org_repo('foo/bar')
+    ('foo', 'bar')
+    >>> split_org_repo('https://api.github.com/repos/foo/bar/')
+    ('foo', 'bar')
+    >>> split_org_repo('github.com/foo/bar/')
+    ('foo', 'bar')
+    >>> split_org_repo('git://github.com/foo/bar.git')
+    ('foo', 'bar')
+    """
+    segments = [s.strip() for s in url_like.split("/") if s.strip()]
+    if not len(segments) >= 2:
+        raise ValueError(f"Not a GitHub-like URL: {url_like}")
+    org = segments[-2]
+    name = segments[-1]
+    if name.endswith(".git"):
+        name, _, _ = name.rpartition(".git")
+    return org, name
+
+
+# FIXME duplicated with purl2vcs.find_source_repo.get_tags_and_commits_from_git_output
+def get_tags_and_commits_from_git_output(git_ls_remote):
+    """
+    Yield tuples of (tag, commit), given a git ls-remote output
+    """
+    for line in git_ls_remote.split("\n"):
+        # line: kjwfgeklngelkfjofjeo123   refs/tags/1.2.3
+        line_segments = line.split("\t")
+        # segments: ["kjwfgeklngelkfjofjeo123", "refs/tags/1.2.3"]
+        if len(line_segments) > 1 and line_segments[1].startswith("refs/tags/"):
+            commit = line_segments[0]
+            tag = line_segments[1].replace("refs/tags/", "")
+            yield tag, commit

minecode_pipelines/pipelines/mine_composer.py

@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+import os
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import federatedcode
+
+from minecode_pipelines import pipes
+from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO
+from minecode_pipelines.pipes.composer import mine_composer_packages
+from minecode_pipelines.pipes.composer import mine_and_publish_composer_purls
+
+MINECODE_COMPOSER_GIT_URL = os.environ.get(
+    "MINECODE_COMPOSER_GIT_URL", "https://github.com/aboutcode-data/minecode-data-composer-test"
+)
+
+
+class MineComposer(Pipeline):
+    """
+    Mine all packageURLs from a composer index and publish them to a FederatedCode repo.
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.clone_composer_repo,
+            cls.mine_and_publish_composer_purls,
+        )
+
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
+
+    def clone_composer_repo(self):
+        """
+        Clone the federatedcode composer url and return the Repo object
+        """
+        self.cloned_data_repo = federatedcode.clone_repository(MINECODE_COMPOSER_GIT_URL)
+        self.cloned_config_repo = federatedcode.clone_repository(MINECODE_PIPELINES_CONFIG_REPO)
+
+    def mine_and_publish_composer_purls(self):
+        """
+        Mine Composer package names from Composer indexes and generate
+        package URLs (pURLs) for all mined Composer packages.
+        """
+
+        composer_packages = mine_composer_packages()
+        mine_and_publish_composer_purls(
+            packages=composer_packages,
+            cloned_data_repo=self.cloned_data_repo,
+            cloned_config_repo=self.cloned_config_repo,
+            logger=self.log,
+        )
+
+    def delete_cloned_repos(self):
+        pipes.delete_cloned_repos(
+            repos=[self.cloned_data_repo, self.cloned_config_repo],
+            logger=self.log,
+        )