Merge pull request #848 from googlefonts/packager2

gftools packager: refactor

Author: m4rc1e

Lib/gftools/actions/updateupstream.py

@@ -1,4 +1,4 @@
-"""Generates an upstream.yaml from a config.yaml and a GitHub release URL
+"""Generates a METADATA.pb from a config.yaml and a GitHub release URL
 
 """
 import argparse
@@ -7,11 +7,13 @@
 import yaml
 import zipfile
 
-import gftools.packager
+from gftools.packager import load_metadata
+import gftools.fonts_public_pb2 as fonts_pb2
 from gftools.builder import GFBuilder
-from strictyaml import as_document
 from gftools.utils import download_file
 from fontTools.ttLib import TTFont
+import gftools.util.google_fonts as fonts
+from copy import deepcopy
 
 
 parser = argparse.ArgumentParser(description="Create an upstream.yaml for a family")
@@ -26,71 +28,84 @@ def get_family_name(config):
     return GFBuilder(config).get_family_name()
 
 
-def generate_upstream(config, url):
+def generate_metadata(config, url):
     repo = os.environ.get("GITHUB_REPOSITORY")
     if not repo:
         raise ValueError("Not being run from a GitHub action?")
     if "category" not in config:
         config["category"] = ["SANS_SERIF"]
 
-    upstream = {
-        "name": get_family_name(config),
-        "repository_url": os.environ["GITHUB_SERVER_URL"] + "/" + repo + ".git",
-        "archive": url,
-        "branch": "main",
-        "category": config["category"],
-        "build": "",
-        "files": {},
-        "designer": "Will be filled in",
-    }
-    return upstream
+    metadata = fonts_pb2.FamilyProto()
+    metadata.name = get_family_name(config)
+    metadata.source.repository_url = os.environ["GITHUB_SERVER_URL"] + "/" + repo
+    metadata.source.archive_url = url
+    metadata.source.branch = "main"
+    metadata.category.append(config["category"][0])
+    metadata.designer = "Will be filled in"
+    return metadata
 
 
-def update_file_list(upstream):
+def update_file_list(metadata):
     print("Downloading release archive...")
-    upstream["files"] = {}
     with TemporaryDirectory() as tmp:
         archive_path = os.path.join(tmp, "archive.zip")
-        download_file(upstream["archive"], archive_path)
+        download_file(metadata.source.archive_url, archive_path)
         license_found = False
         description_found = False
         a_font = None
         with zipfile.ZipFile(archive_path, "r") as zip_ref:
             zip_ref.extractall(tmp)
+        metadata.source.files.clear()
         for root, _, files in os.walk(tmp):
             for file in files:
                 fullpath = os.path.join(root, file)
                 relpath = os.path.relpath(fullpath, tmp)
                 print(relpath)
+                item = fonts_pb2.SourceFileProto()
                 if file == "OFL.txt":
                     license_found = True
-                    upstream["files"][relpath] = file
+                    item.source_file = relpath
+                    item.dest_file = file
+                    metadata.source.files.append(item)
                 elif file == "DESCRIPTION.en_us.html":
                     description_found = True
-                    upstream["files"][relpath] = file
+                    metadata["files"][relpath] = file
+                    item.source_file = relpath
+                    item.dest_file = file
+                    metadata.source.files.append(item)
                 elif file == "ARTICLE.en_us.html":
-                    upstream["files"][relpath] = "article/"+file
+                    item.source_file = relpath
+                    item.dest_file = "article/"+file
+                    metadata.source.files.append(item)
                 elif file.endswith("ttf"):
                     if config.get("buildVariable", True):
                         # Only add the file if it is the variable font
                         if "[" in file:
-                            upstream["files"][relpath] = file
+                            item.source_file = relpath
+                            item.dest_file = file
+                            metadata.source.files.append(item)
                             a_font = fullpath
                     else:
                         # Add statics
-                        upstream["files"][relpath] = file
+                        item.source_file = relpath
+                        item.dest_file = file
+                        metadata.source.files.append(item)
                         a_font = fullpath
 
         # If there was a "googlefonts/" directory in the release, just
         # use files in that directory.
-        if any("googlefonts/" in x for x in upstream["files"].keys()):
-            upstream["files"] = {str(k):str(v) for k,v in upstream["files"].items() if "googlefonts/" in str(k) or not ".ttf" in str(k) }
+        if any("googlefonts/" in x.source_file for x in metadata.source.files):
+            existing_files = deepcopy(metadata.source.files)
+            metadata.source.files.clear()
+            for item in existing_files:
+                if "googlefonts/" in item.source_file or not ".ttf" in item.source_file:
+                    metadata.source.files.append(item)
 
         if not license_found:
             raise ValueError(
                 "No license file was found. Ensure OFL.txt is added the the release"
             )
-        if not description_found and "Noto" not in upstream["name"]:
+        if not description_found and "Noto" not in metadata.name:
             raise ValueError(
                 "No description file was found. Ensure DESCRIPTION.en_us.html is added the the release"
             )
@@ -100,10 +115,10 @@ def update_file_list(upstream):
             raise ValueError("No font files were found. Is the release broken?")
 
         designer = TTFont(a_font)["name"].getDebugName(9)
-        if "Noto" in upstream["name"]:
-            upstream["designer"] = "Google"
+        if "Noto" in metadata.name:
+            metadata.designer = "Google"
         elif designer:
-            upstream["designer"] = designer
+            metadata.designer = designer
 
 
 if __name__ == "__main__":
@@ -115,28 +130,18 @@ def update_file_list(upstream):
             open(args.config), Loader=yaml.FullLoader
         )
 
-    if os.path.isfile("upstream.yaml"):
-        try:
-            upstream = gftools.packager._upstream_conf_from_file(
-                "upstream.yaml", yes=True, quiet=True
-            )
-        except Exception as e:
-            raise ValueError("Something went wrong parsing upstream.yaml: " + str(e))
+    if os.path.isfile("METADATA.pb"):
+        metadata = load_metadata("METADATA.pb")
     else:
         try:
-            upstream = as_document(
-                generate_upstream(config, args.url),
-                gftools.packager.upstream_yaml_schema,
-            )
+            metadata = generate_metadata(config, args.url)
         except Exception as e:
             raise ValueError(
-                "Something went wrong generating upstream.yaml (bug in updateupstream): "
+                "Something went wrong generating METADATA.pb (bug in updateupstream): "
                 + str(e)
             )
 
     # Add archive URL
-    upstream["archive"] = args.url
-    update_file_list(upstream)
-
-    with open("upstream.yaml", "w") as upstream_yaml_file:
-        upstream_yaml_file.write(upstream.as_yaml())
+    metadata.source.archive_url = args.url
+    update_file_list(metadata)
+    fonts.WriteProto(metadata, "METADATA.pb")

Lib/gftools/builder/schema.py

@@ -15,7 +15,8 @@
     Any,
     MapCombined,
 )
-from gftools.packager import CATEGORIES
+
+CATEGORIES = ['DISPLAY', 'SERIF', 'SANS_SERIF', 'HANDWRITING', 'MONOSPACE']
 
 
 BASE_SCHEMA = MapCombined(

Lib/gftools/gfgithub.py

@@ -56,6 +56,19 @@ def rest_url(self, path, **kwargs):
             )
         return base_url
 
+    def get_content(self, path, branch=None):
+        if branch:
+            url = self.rest_url(f"contents/{path}", ref=branch)
+        else:
+            url = self.rest_url(f"contents/{path}")
+        headers = {
+            "Accept": "application/vnd.github.v3.raw",
+            "Authorization": f"bearer {self.gh_token}",
+        }
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+        return response
+
     def get_blob(self, file_sha):
         url = self.rest_url(f"git/blobs/{file_sha}")
         headers = {
@@ -66,6 +79,9 @@ def get_blob(self, file_sha):
         response.raise_for_status()
         return response
 
+    def get_latest_release(self):
+        return self._get(self.rest_url("releases/latest"))
+
     def open_prs(self, pr_head: str, pr_base_branch: str) -> typing.List:
         return self._get(
             self.rest_url("pulls", state="open", head=pr_head, base=pr_base_branch)