Generate cross compiled release files and make sure that release remote files have the right hashes (#44)

tiagovtristao · web-flow · commit 2456a163deab · 2022-03-03T15:43:12.000Z
* Generate cross compiled release files and make sure that release remote files have the right hashes

* Fixes

* Bump plz

* Fix
diff --git a/.github/workflows/plugin.yaml b/.github/workflows/plugin.yaml
@@ -28,8 +28,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
-      - name: build
-        run: ./pleasew build //tools/...
+      - name: Test
+        run: ./pleasew test //test/release_remote_files:please_pex_remote_hashes
+      - name: Build
+        run: ./pleasew build //package:release_files
       - name: Release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.plzconfig b/.plzconfig
@@ -1,5 +1,8 @@
 [Please]
-Version = >=16.17.0
+Version = >=16.19.0
+
+[Build]
+hashcheckers = sha256
 
 [Parse]
 PreloadBuildDefs = test/build_defs/test.build_defs
@@ -21,6 +24,7 @@ DefaultValue = python3
 
 [PluginConfig "pex_tool"]
 ConfigKey = PexTool
+; Release artifact of //tools/please_pex:pex_main.
 DefaultValue = //tools:please_pex
 
 [PluginConfig "interpreter_options"]
@@ -83,3 +87,4 @@ Optional = true
 [featureflags]
 PythonWheelHashing = true
 ExcludePythonRules = true
+SingleSHA1Hash = true
diff --git a/BUILD b/BUILD
@@ -1,7 +1,12 @@
+subinclude("//build_defs:version")
+
 filegroup(
     name = "pleasew",
     srcs = ["pleasew"],
     binary = True,
     test_only = True,
     visibility = ["PUBLIC"],
 )
+
+version(name = "version")
+
diff --git a/build_defs/BUILD b/build_defs/BUILD
@@ -1,11 +1,30 @@
+filegroup(
+    name = "archs",
+    srcs = ["archs.build_defs"],
+    visibility = ["PUBLIC"],
+)
+
 filegroup(
     name = "python",
     srcs = ["python.build_defs"],
     visibility = ["PUBLIC"],
 )
 
+filegroup(
+    name = "version",
+    srcs = ["version.build_defs"],
+    visibility = ["PUBLIC"],
+)
+
 filegroup(
     name = "multiversion_wheel",
     srcs = ["multiversion_wheel.build_defs"],
     visibility = ["//third_party/..."],
 )
+
+filegroup(
+    name = "release_file",
+    srcs = ["release_file.build_defs"],
+    visibility = ["//package/..."],
+)
+
diff --git a/build_defs/archs.build_defs b/build_defs/archs.build_defs
@@ -0,0 +1,7 @@
+SUPPORTED_ARCHS = [
+    "darwin_amd64",
+    "darwin_arm64",
+    "freebsd_amd64",
+    "linux_amd64",
+    "linux_arm64",
+]
diff --git a/build_defs/release_file.build_defs b/build_defs/release_file.build_defs
@@ -0,0 +1,11 @@
+subinclude("//:version")
+
+def release_file(name:str, file_target:str, arch:str):
+    return genrule(
+        name = f"{name}_{arch}",
+        srcs = [f"///{arch}" + canonicalise(file_target)],
+        outs = [f"{name}-{VERSION}-{arch}"],
+        cmd = "mv $SRC $OUT",
+        labels = [f"release-file:{name}"],
+    )
+
diff --git a/build_defs/version.build_defs b/build_defs/version.build_defs
@@ -0,0 +1,9 @@
+def version(name:str, version_file:str="VERSION", visibility:list=["PUBLIC"]):
+    return genrule(
+    name = name,
+    srcs = [version_file],
+    outs = [f"{name}.build_defs"],
+    cmd = "echo VERSION = \\\"$(cat $SRCS)\\\" > $OUT",
+    visibility = visibility,
+)
+
diff --git a/package/BUILD b/package/BUILD
@@ -0,0 +1,18 @@
+export_file(
+    name = "files",
+    src = "files.build_defs",
+    visibility = ["PUBLIC"],
+)
+
+subinclude(":files", "//build_defs:release_file", "//build_defs:archs")
+
+filegroup(
+    name = "release_files",
+    srcs = [
+        release_file(name, target, arch)
+        for name, target in RELEASE_FILES.items()
+        for arch in SUPPORTED_ARCHS
+    ],
+    labels = ["hlink:plz-out/package"],
+)
+
diff --git a/package/files.build_defs b/package/files.build_defs
@@ -0,0 +1,4 @@
+RELEASE_FILES = {
+    "please_pex": "//tools/please_pex:pex_main",
+}
+
diff --git a/test/build_defs/BUILD b/test/build_defs/BUILD
@@ -0,0 +1,5 @@
+export_file(
+    name = "release_remote_file",
+    src = "release_remote_file.build_defs",
+    visibility = ["//test/..."],
+)
diff --git a/test/build_defs/release_remote_file.build_defs b/test/build_defs/release_remote_file.build_defs
@@ -0,0 +1,47 @@
+# TODO(tiagovtritao): There seems to be some sort of lock going on preventing this subinclude from working.
+# subinclude("//package:files")
+
+def release_remote_file_hash_test(name:str, release_file:str, remote_file_label:str):
+    """
+    Tests whether a `remote_file` target downloading a release file within this repo has matching hashes.
+    """
+
+    # if not RELEASE_FILES.get(release_file):
+    #     fail(f"'{release_file}' is not a release file")
+
+    test_cmd = " && ".join([
+        # Build release file.
+        f"plz -i release-file:{release_file} build //package/... >/dev/null 2>&1",
+        # Get sha256 hashes for release file for all supported architectures.
+        "REPO_ROOT=$(plz query reporoot)",
+        "plz -i release-file:%s query outputs //package/... | xargs -L1 -I{} sha256sum $REPO_ROOT/{} | cut -f1 -d' ' | sort > new_hashes" % release_file,
+        # Get current hashes for `remote_file` that downloads this release file.
+        f"plz query print {remote_file_label} --field=hashes | sort > old_hashes",
+
+        # Check if `remote_file` hashes need updating.
+        f"if [ \"$(cat new_hashes)\" != \"$(cat old_hashes)\" ]; then "
+            f"echo \"You need to update the hashes of {remote_file_label} with:\"; "
+            "cat new_hashes; "
+            "exit 1; "
+        "fi"
+    ])
+
+    def _test_cmd(cmd):
+        args = '$PLZ_ARGS -o sandbox.test:false -o sandbox.build:false -o sandbox.namespace:false '
+        return cmd.replace('plz ', f'$TOOLS --nolock {args} -o cache.dirclean:false --log_file plz-out/log/{name}.log ')
+
+    return gentest(
+        name = name,
+        test_cmd = {
+            'opt': _test_cmd(test_cmd),
+            'dbg': _test_cmd(test_cmd.replace('plz ', 'plz -c dbg ')),
+            'cover': _test_cmd(test_cmd.replace('plz ', 'plz -c cover ')),
+        },
+        test_tools = ["//:pleasew"],
+        labels = ["manual"],
+        no_test_output = True,
+        sandbox = False,
+        local = True,
+        pass_env = ["PLZ_ARGS"],
+        exit_on_error = True,
+    )
diff --git a/test/release_remote_files/BUILD b/test/release_remote_files/BUILD
@@ -0,0 +1,8 @@
+subinclude("//test/build_defs:release_remote_file")
+
+release_remote_file_hash_test(
+    name = "please_pex_remote_hashes",
+    release_file = "please_pex",
+    remote_file_label = "//tools:please_pex",
+)
+
diff --git a/third_party/go/BUILD b/third_party/go/BUILD
@@ -1,3 +1,5 @@
+subinclude("//build_defs:archs")
+
 go_toolchain(
     name = "toolchain",
     hashes = [
@@ -6,6 +8,7 @@ go_toolchain(
         "6bf89fc4f5ad763871cf7eac80a2d594492de7a818303283f1366a7f6a30372d",  # linux_amd64
         "15c184c83d99441d719da201b26256455eee85a808747c404b4183e9aa6c64b4",  # freebsd_amd64
     ],
+    architectures = SUPPORTED_ARCHS,
     strip_srcs = CONFIG.BUILD_CONFIG != "dbg",
     version = "1.17",
 )
diff --git a/tools/BUILD b/tools/BUILD
@@ -1,22 +1,26 @@
+subinclude("//:version")
+
+# This target is meant to be used as the default pex tool for this plugin. It relies on the released artifact which might
+# not exist yet and as such, the hash sums won't be available too. To solve this a test will be run to guarantee that the
+# hashes below will match the ones of the artifact once released.
 remote_file(
     name = "please_pex",
-    url = "https://github.com/please-build/python-rules/releases/download/v0.4.0/please_pex",
-    extract = False,
+    url = f"https://github.com/please-build/python-rules/releases/download/v{VERSION}/please_pex-{VERSION}-{CONFIG.HOSTOS}_{CONFIG.HOSTARCH}",
+    hashes = [
+        "28abdae0d17ca666663563f9b874ddfae0c5209389271d933945003c0ce9d6ab",
+        "3c621e8ff11301b64c7a0ca84ba23161fd5c777e4df82034060925f5106ee317",
+        "917e6ff3591f6bca5cfc2a251f136c180d743856bee877dc18626cc135bbe7a0",
+        "9c20d867ec1a1fcea85481d4a29ec80f8af5923fc06c7ad0057105918ec3e236",
+        "ab3426a82ab4cb5c5b0a4979a4a3821eb10361726759e30ab5f2a1e7c09fc78e",
+    ],
     binary = True,
     visibility = ["PUBLIC"],
-    out = "please_pex_tool",
 )
 
-# remote_file(
-#     name = "wheel_resolver",
-#     url = "https://www.github.com/please-build/python_rules/...",
-# )
-
 remote_file(
     name = "arcat",
-    url = "https://github.com/please-build/arcat/releases/download/v1.0.0/arcat-1.0.0-linux_amd64",
+    url = f"https://github.com/please-build/arcat/releases/download/v1.0.0/arcat-1.0.0-{CONFIG.HOSTOS}_{CONFIG.HOSTARCH}",
     hashes = ["6705304dde3cf358d7b8f5a034dfcc8242474b76d0e439b2f39a7a99f8612749"],
-    extract = False,
     binary = True,
     visibility = ["PUBLIC"],
 )
diff --git a/tools/please_pex/BUILD b/tools/please_pex/BUILD
@@ -1,12 +1,5 @@
 subinclude("//build_defs:python")
 
-filegroup(
-    name = "please_pex",
-    srcs = [":pex_main"],
-    visibility = ["PUBLIC"],
-    labels = ["hlink:plz-out/package"],
-)
-
 go_binary(
     name = "pex_main",
     srcs = ["pex_main.go"],
@@ -15,6 +8,7 @@ go_binary(
         "//third_party/go:go-logging.v1",
         "//tools/please_pex/pex",
     ],
+    visibility = ["//package/..."],
 )
 
 python_test(

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +RELEASE_FILES = {
 +    "please_pex": "//tools/please_pex:pex_main",
 +}
++