fix(py_wheel): produce deterministic wheel files (#1453)

Current implementation does not produce deterministic output because:
- `ZipFile.writestr()` leaks current date and time
- `ZipFile.write()` leaks the source file's mtime and mode bits
(permissions) into the resulting zip archive.

By manually creating our own `ZipInfo` objects we can explicitly set
date and time fields to `Jan 1, 1980, 00:00` (minimum value allowed by
the zip file standard), and ensure that other file attributes are
uniform across all entries in a zip file.

---------

Co-authored-by: Ignas Anikevicius <[email protected]>

Author: dizzy57

CHANGELOG.md

@@ -105,6 +105,8 @@ A brief description of the categories of changes:
 
 * (gazelle) Improve runfiles lookup hermeticity.
 
+* (py_wheel) Produce deterministic wheel files
+
 ## [0.25.0] - 2023-08-22
 
 ### Changed

examples/wheel/wheel_test.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
 import os
 import platform
 import subprocess
@@ -43,9 +44,29 @@ def _get_path(self, filename):
         else:
             return path
 
+    def assertFileSha256Equal(self, filename, sha):
+        hash = hashlib.sha256()
+        with open(filename, "rb") as f:
+            while True:
+                buf = f.read(2**20)
+                if not buf:
+                    break
+                hash.update(buf)
+        self.assertEqual(hash.hexdigest(), sha)
+
+    def assertAllEntriesHasReproducibleMetadata(self, zf):
+        for zinfo in zf.infolist():
+            self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0), msg=zinfo.filename)
+            self.assertEqual(zinfo.create_system, 3, msg=zinfo.filename)
+            self.assertEqual(zinfo.external_attr, 0o777 << 16, msg=zinfo.filename)
+            self.assertEqual(
+                zinfo.compress_type, zipfile.ZIP_DEFLATED, msg=zinfo.filename
+            )
+
     def test_py_library_wheel(self):
         filename = self._get_path("example_minimal_library-0.0.1-py3-none-any.whl")
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             self.assertEqual(
                 zf.namelist(),
                 [
@@ -56,12 +77,16 @@ def test_py_library_wheel(self):
                     "example_minimal_library-0.0.1.dist-info/RECORD",
                 ],
             )
+        self.assertFileSha256Equal(
+            filename, "6da8e06a3fdd9ae5ee9fa8f796610723c05a4b0d7fde0ec5179401e956204139"
+        )
 
     def test_py_package_wheel(self):
         filename = self._get_path(
             "example_minimal_package-0.0.1-py3-none-any.whl",
         )
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             self.assertEqual(
                 zf.namelist(),
                 [
@@ -74,12 +99,16 @@ def test_py_package_wheel(self):
                     "example_minimal_package-0.0.1.dist-info/RECORD",
                 ],
             )
+        self.assertFileSha256Equal(
+            filename, "2948b0b5e0aa421e0b40f78b74018bbc2f218165f211da0a4609e431e8e52bee"
+        )
 
     def test_customized_wheel(self):
         filename = self._get_path(
             "example_customized-0.0.1-py3-none-any.whl",
         )
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             self.assertEqual(
                 zf.namelist(),
                 [
@@ -159,12 +188,16 @@ def test_customized_wheel(self):
 first = first.main:f
 second = second.main:s""",
             )
+        self.assertFileSha256Equal(
+            filename, "66f0c1bfe2cedb2f4cf08d4fe955096860186c0a2f3524e0cb02387a55ac3e63"
+        )
 
     def test_legacy_filename_escaping(self):
         filename = self._get_path(
             "file_name_escaping-0.0.1_r7-py3-none-any.whl",
         )
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             self.assertEquals(
                 zf.namelist(),
                 [
@@ -193,6 +226,9 @@ def test_legacy_filename_escaping(self):
 UNKNOWN
 """,
             )
+        self.assertFileSha256Equal(
+            filename, "593c6ab58627f2446d0f1ef2956fd6d42104eedce4493c72d462f7ebf8cb74fa"
+        )
 
     def test_filename_escaping(self):
         filename = self._get_path(
@@ -234,6 +270,7 @@ def test_custom_package_root_wheel(self):
         )
 
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             self.assertEqual(
                 zf.namelist(),
                 [
@@ -255,13 +292,17 @@ def test_custom_package_root_wheel(self):
             # Ensure RECORD files do not have leading forward slashes
             for line in record_contents.splitlines():
                 self.assertFalse(line.startswith("/"))
+        self.assertFileSha256Equal(
+            filename, "1b1fa3a4e840211084ef80049d07947b845c99bedb2778496d30e0c1524686ac"
+        )
 
     def test_custom_package_root_multi_prefix_wheel(self):
         filename = self._get_path(
             "example_custom_package_root_multi_prefix-0.0.1-py3-none-any.whl",
         )
 
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             self.assertEqual(
                 zf.namelist(),
                 [
@@ -282,13 +323,17 @@ def test_custom_package_root_multi_prefix_wheel(self):
             # Ensure RECORD files do not have leading forward slashes
             for line in record_contents.splitlines():
                 self.assertFalse(line.startswith("/"))
+        self.assertFileSha256Equal(
+            filename, "f0422d7a338de3c76bf2525927fd93c0f47f2e9c60ecc0944e3e32b642c28137"
+        )
 
     def test_custom_package_root_multi_prefix_reverse_order_wheel(self):
         filename = self._get_path(
             "example_custom_package_root_multi_prefix_reverse_order-0.0.1-py3-none-any.whl",
         )
 
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             self.assertEqual(
                 zf.namelist(),
                 [
@@ -309,12 +354,16 @@ def test_custom_package_root_multi_prefix_reverse_order_wheel(self):
             # Ensure RECORD files do not have leading forward slashes
             for line in record_contents.splitlines():
                 self.assertFalse(line.startswith("/"))
+        self.assertFileSha256Equal(
+            filename, "4f9e8c917b4050f121ac81e9a2bb65723ef09a1b90b35d93792ac3a62a60efa3"
+        )
 
     def test_python_requires_wheel(self):
         filename = self._get_path(
             "example_python_requires_in_a_package-0.0.1-py3-none-any.whl",
         )
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             metadata_contents = zf.read(
                 "example_python_requires_in_a_package-0.0.1.dist-info/METADATA"
             )
@@ -330,6 +379,9 @@ def test_python_requires_wheel(self):
 UNKNOWN
 """,
             )
+        self.assertFileSha256Equal(
+            filename, "9bfe8197d379f88715458a75e45c1f521a8b9d3cc43fe19b407c4ab207228b7c"
+        )
 
     def test_python_abi3_binary_wheel(self):
         arch = "amd64"
@@ -346,6 +398,7 @@ def test_python_abi3_binary_wheel(self):
             f"example_python_abi3_binary_wheel-0.0.1-cp38-abi3-{os_string}_{arch}.whl",
         )
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             metadata_contents = zf.read(
                 "example_python_abi3_binary_wheel-0.0.1.dist-info/METADATA"
             )
@@ -380,6 +433,7 @@ def test_rule_creates_directory_and_is_included_in_wheel(self):
         )
 
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             self.assertEqual(
                 zf.namelist(),
                 [
@@ -390,13 +444,17 @@ def test_rule_creates_directory_and_is_included_in_wheel(self):
                     "use_rule_with_dir_in_outs-0.0.1.dist-info/RECORD",
                 ],
             )
+        self.assertFileSha256Equal(
+            filename, "8ad5f639cc41ac6ac67eb70f6553a7fdecabaf3a1b952c3134eaea59610c2a64"
+        )
 
     def test_rule_expands_workspace_status_keys_in_wheel_metadata(self):
         filename = self._get_path(
             "example_minimal_library_BUILD_USER_-0.1._BUILD_TIMESTAMP_-py3-none-any.whl"
         )
 
         with zipfile.ZipFile(filename) as zf:
+            self.assertAllEntriesHasReproducibleMetadata(zf)
             metadata_file = None
             for f in zf.namelist():
                 self.assertNotIn("_BUILD_TIMESTAMP_", f)

tools/wheelmaker.py

@@ -14,14 +14,15 @@
 
 import argparse
 import base64
-import collections
 import hashlib
 import os
 import re
 import sys
 import zipfile
 from pathlib import Path
 
+_ZIP_EPOCH = (1980, 1, 1, 0, 0, 0)
+
 
 def commonpath(path1, path2):
     ret = []
@@ -189,7 +190,8 @@ def add_string(self, filename, contents):
         """Add given 'contents' as filename to the distribution."""
         if sys.version_info[0] > 2 and isinstance(contents, str):
             contents = contents.encode("utf-8", "surrogateescape")
-        self._zipfile.writestr(filename, contents)
+        zinfo = self._zipinfo(filename)
+        self._zipfile.writestr(zinfo, contents)
         hash = hashlib.sha256()
         hash.update(contents)
         self._add_to_record(filename, self._serialize_digest(hash), len(contents))
@@ -219,20 +221,36 @@ def arcname_from(name):
             return
 
         arcname = arcname_from(package_filename)
+        zinfo = self._zipinfo(arcname)
 
-        self._zipfile.write(real_filename, arcname=arcname)
-        # Find the hash and length
+        # Write file to the zip archive while computing the hash and length
         hash = hashlib.sha256()
         size = 0
-        with open(real_filename, "rb") as f:
-            while True:
-                block = f.read(2**20)
-                if not block:
-                    break
-                hash.update(block)
-                size += len(block)
+        with open(real_filename, "rb") as fsrc:
+            with self._zipfile.open(zinfo, "w") as fdst:
+                while True:
+                    block = fsrc.read(2**20)
+                    if not block:
+                        break
+                    fdst.write(block)
+                    hash.update(block)
+                    size += len(block)
         self._add_to_record(arcname, self._serialize_digest(hash), size)
 
+    def _zipinfo(self, filename):
+        """Construct deterministic ZipInfo entry for a file named filename"""
+        # Strip leading path separators to mirror ZipInfo.from_file behavior
+        separators = os.path.sep
+        if os.path.altsep is not None:
+            separators += os.path.altsep
+        arcname = filename.lstrip(separators)
+
+        zinfo = zipfile.ZipInfo(filename=arcname, date_time=_ZIP_EPOCH)
+        zinfo.create_system = 3  # ZipInfo entry created on a unix-y system
+        zinfo.external_attr = 0o777 << 16  # permissions: rwxrwxrwx
+        zinfo.compress_type = self._zipfile.compression
+        return zinfo
+
     def add_wheelfile(self):
         """Write WHEEL file to the distribution"""
         # TODO(pstradomski): Support non-purelib wheels.