Merge pull request #45 from nexB/open-as-text

Open as text

Author: pombredanne

configure

@@ -51,7 +51,10 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
-PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty --find-links https://thirdparty.aboutcode.org/pypi"
+if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then
+    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty "
+fi
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi"
 
 
 ################################

configure.bat

@@ -49,7 +49,11 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 
 @rem ################################
 @rem # Thirdparty package locations and index handling
-set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG%
+if exist ""%CFG_ROOT_DIR%\thirdparty"" (
+    set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty "
+)
+
+set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG%
 @rem ################################
 
 

etc/scripts/publish_files.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import hashlib
+import os
+import sys
+
+from pathlib import Path
+
+import click
+import requests
+import utils_thirdparty
+
+from github_release_retry import github_release_retry as grr
+
+"""
+Create GitHub releases and upload files there.
+"""
+
+
+def get_files(location):
+    """
+    Return an iterable of (filename, Path, md5) tuples for files in the `location`
+    directory tree recursively.
+    """
+    for top, _dirs, files in os.walk(location):
+        for filename in files:
+            pth = Path(os.path.join(top, filename))
+            with open(pth, 'rb') as fi:
+                md5 = hashlib.md5(fi.read()).hexdigest()
+            yield filename, pth, md5
+
+
+def get_etag_md5(url):
+    """
+    Return the cleaned etag of URL `url` or None.
+    """
+    headers = utils_thirdparty.get_remote_headers(url)
+    headers = {k.lower(): v for k, v in headers.items()}
+    etag = headers .get('etag')
+    if etag:
+        etag = etag.strip('"').lower()
+        return etag
+
+
+def create_or_update_release_and_upload_directory(
+        user,
+        repo,
+        tag_name,
+        token,
+        directory,
+        retry_limit=10,
+        description=None,
+):
+    """
+    Create or update a GitHub release at https://github.com/<user>/<repo> for
+    `tag_name` tag using the optional `description` for this release.
+    Use the provided `token` as a GitHub token for API calls authentication.
+    Upload all files found in the `directory` tree to that GitHub release.
+    Retry API calls up to `retry_limit` time to work around instability the
+    GitHub API.
+
+    Remote files that are not the same as the local files are deleted and re-
+    uploaded.
+    """
+    release_homepage_url = f'https://github.com/{user}/{repo}/releases/{tag_name}'
+
+    # scrape release page HTML for links
+    urls_by_filename = {os.path.basename(l): l
+        for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url)
+    }
+
+    # compute what is new, modified or unchanged
+    print(f'Compute which files is new, modified or unchanged in {release_homepage_url}')
+
+    new_to_upload = []
+    unchanged_to_skip = []
+    modified_to_delete_and_reupload = []
+    for filename, pth, md5 in get_files(directory):
+        url = urls_by_filename.get(filename)
+        if not url:
+            print(f'{filename} content is NEW, will upload')
+            new_to_upload.append(pth)
+            continue
+
+        out_of_date = get_etag_md5(url) != md5
+        if out_of_date:
+            print(f'{url} content is CHANGED based on md5 etag, will re-upload')
+            modified_to_delete_and_reupload.append(pth)
+        else:
+            # print(f'{url} content is IDENTICAL, skipping upload based on Etag')
+            unchanged_to_skip.append(pth)
+            print('.')
+
+    ghapi = grr.GithubApi(
+        github_api_url='https://api.github.com',
+        user=user,
+        repo=repo,
+        token=token,
+        retry_limit=retry_limit,
+    )
+
+    # yank modified
+    print(
+        f'Unpublishing {len(modified_to_delete_and_reupload)} published but '
+        f'locally modified files in {release_homepage_url}')
+
+    release = ghapi.get_release_by_tag(tag_name)
+
+    for pth in modified_to_delete_and_reupload:
+        filename = os.path.basename(pth)
+        asset_id = ghapi.find_asset_id_by_file_name(filename, release)
+        print (f'  Unpublishing file: {filename}).')
+        response = ghapi.delete_asset(asset_id)
+        if response.status_code != requests.codes.no_content:  # NOQA
+            raise Exception(f'failed asset deletion: {response}')
+
+    # finally upload new and modified
+    to_upload = new_to_upload + modified_to_delete_and_reupload
+    print(f'Publishing with {len(to_upload)} files to {release_homepage_url}')
+    release = grr.Release(tag_name=tag_name, body=description)
+    grr.make_release(ghapi, release, to_upload)
+
+
+TOKEN_HELP = (
+    'The Github personal acess token is used to authenticate API calls. '
+    'Required unless you set the GITHUB_TOKEN environment variable as an alternative. '
+    'See for details: https://github.com/settings/tokens and '
+    'https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token'
+)
+
+
+@click.command()
+
+@click.option(
+    '--user-repo-tag',
+    help='The GitHub qualified repository user/name/tag in which '
+        'to create the release such as in nexB/thirdparty/pypi',
+    type=str,
+    required=True,
+)
+@click.option(
+    '-d', '--directory',
+    help='The directory that contains files to upload to the release.',
+    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True),
+    required=True,
+)
+@click.option(
+    '--token',
+    help=TOKEN_HELP,
+    default=os.environ.get('GITHUB_TOKEN', None),
+    type=str,
+    required=False,
+)
+@click.option(
+    '--description',
+    help='Text description for the release. Ignored if the release exists.',
+    default=None,
+    type=str,
+    required=False,
+)
+@click.option(
+    '--retry_limit',
+    help='Number of retries when making failing GitHub API calls. '
+        'Retrying helps work around transient failures of the GitHub API.',
+    type=int,
+    default=10,
+)
+@click.help_option('-h', '--help')
+def publish_files(
+    user_repo_tag,
+    directory,
+    retry_limit=10, token=None, description=None,
+):
+    """
+    Publish all the files in DIRECTORY as assets to a GitHub release.
+    Either create or update/replace remote files'
+    """
+    if not token:
+        click.secho('--token required option is missing.')
+        click.secho(TOKEN_HELP)
+        sys.exit(1)
+
+    user, repo, tag_name = user_repo_tag.split('/')
+
+    create_or_update_release_and_upload_directory(
+        user=user,
+        repo=repo,
+        tag_name=tag_name,
+        description=description,
+        retry_limit=retry_limit,
+        token=token,
+        directory=directory,
+    )
+
+
+if __name__ == '__main__':
+    publish_files()

etc/scripts/utils_thirdparty.py

@@ -24,13 +24,14 @@
 import attr
 import license_expression
 import packageurl
-import utils_pip_compatibility_tags
-import utils_pypi_supported_tags
 import requests
 import saneyaml
+import utils_pip_compatibility_tags
+import utils_pypi_supported_tags
 
 from commoncode import fileutils
 from commoncode.hash import multi_checksums
+from commoncode.text import python_safe_name
 from packaging import tags as packaging_tags
 from packaging import version as packaging_version
 from utils_requirements import load_requirements
@@ -172,11 +173,20 @@ def fetch_wheels(
     else:
         force_pinned = False
 
-    rrp = list(get_required_remote_packages(
-        requirements_file=requirements_file,
-        force_pinned=force_pinned,
-        remote_links_url=remote_links_url,
-    ))
+    try:
+        rrp = list(get_required_remote_packages(
+            requirements_file=requirements_file,
+            force_pinned=force_pinned,
+            remote_links_url=remote_links_url,
+        ))
+    except Exception as e:
+        raise Exception(
+            dict(
+                requirements_file=requirements_file,
+                force_pinned=force_pinned,
+                remote_links_url=remote_links_url,
+            )
+        ) from e
 
     fetched_filenames = set()
     for name, version, package in rrp:
@@ -211,6 +221,7 @@ def fetch_wheels(
             print(f'Missed package {nv} in remote repo, has only:')
             for pv in rr.get_versions(n):
                 print('  ', pv)
+        raise Exception('Missed some packages in remote repo')
 
 
 def fetch_sources(
@@ -261,6 +272,8 @@ def fetch_sources(
             fetched = package.fetch_sdist(dest_dir=dest_dir)
             error = f'Failed to fetch' if not fetched else None
             yield package, error
+    if missed:
+        raise Exception(f'Missing source packages in {remote_links_url}', missed)
 
 ################################################################################
 #
@@ -693,8 +706,7 @@ def save_if_modified(location, content):
                     return False
 
             if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}')
-            wmode = 'wb' if isinstance(content, bytes) else 'w'
-            with open(location, wmode, encoding="utf-8") as fo:
+            with open(location, 'w') as fo:
                 fo.write(content)
             return True
 
@@ -905,16 +917,16 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR):
         other_classifiers = [c for c in classifiers if not c.startswith('License')]
 
         holder = raw_data['Author']
-        holder_contact=raw_data['Author-email']
-        copyright = f'Copyright (c) {holder} <{holder_contact}>'
+        holder_contact = raw_data['Author-email']
+        copyright_statement = f'Copyright (c) {holder} <{holder_contact}>'
 
         pkginfo_data = dict(
             name=raw_data['Name'],
             declared_license=declared_license,
             version=raw_data['Version'],
             description=raw_data['Summary'],
             homepage_url=raw_data['Home-page'],
-            copyright=copyright,
+            copyright=copyright_statement,
             license_expression=license_expression,
             holder=holder,
             holder_contact=holder_contact,
@@ -1845,7 +1857,7 @@ def get(self, path_or_url, as_text=True):
         if not os.path.exists(cached):
             content = get_file_content(path_or_url=path_or_url, as_text=as_text)
             wmode = 'w' if as_text else 'wb'
-            with open(cached, wmode, encoding="utf-8") as fo:
+            with open(cached, wmode) as fo:
                 fo.write(content)
             return content
         else:
@@ -1857,7 +1869,7 @@ def put(self, filename, content):
         """
         cached = os.path.join(self.directory, filename)
         wmode = 'wb' if isinstance(content, bytes) else 'w'
-        with open(cached, wmode, encoding="utf-8") as fo:
+        with open(cached, wmode) as fo:
             fo.write(content)
 
 
@@ -2331,7 +2343,7 @@ def get_required_remote_packages(
         repo = get_remote_repo(remote_links_url=remote_links_url)
     else:
         # a local path
-        assert os.path.exists(remote_links_url)
+        assert os.path.exists(remote_links_url), f'Path does not exist: {remote_links_url}'
         repo = get_local_repo(directory=remote_links_url)
 
     for name, version in required_name_versions:
@@ -2365,7 +2377,7 @@ def update_requirements(name, version=None, requirements_file='requirements.txt'
         updated_name_versions = sorted(updated_name_versions)
         nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions)
 
-        with open(requirements_file, 'w', encoding="utf-8") as fo:
+        with open(requirements_file, 'w') as fo:
             fo.write(nvs)
 
 
@@ -2383,7 +2395,7 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.t
             raise Exception(f'Missing required package {name}=={version}')
         hashed.append(package.specifier_with_hashes)
 
-    with open(requirements_file, 'w', encoding="utf-8") as fo:
+    with open(requirements_file, 'w') as fo:
         fo.write('\n'.join(hashed))
 
 ################################################################################
@@ -2961,5 +2973,6 @@ def compute_normalized_license_expression(declared_licenses):
         from packagedcode import pypi
         return pypi.compute_normalized_license(declared_licenses)
     except ImportError:
-        # Scancode is not installed, we join all license strings and return it
-        return ' '.join(declared_licenses)
+        # Scancode is not installed, clean and join all the licenses
+        lics = [python_safe_name(l).lower() for l in declared_licenses]
+        return ' AND '.join(lics).lower()