bazel: Upgrade rules_rust and a few other rule sets (#32098)

This PR upgrades our use of `rules_rust` to `v0.59.3`, which is a custom
version I released on our
[fork](https://github.com/MaterializeInc/rules_rust/releases/tag/mz-0.59.3).
As described on the release, it's a fork of the upstream `v0.59.2` and
includes two PRs I made upstream:

1. bazelbuild/rules_rust#3386
2. bazelbuild/rules_rust#3387

In addition to `rules_rust` we upgrade (out of necessity)

* `rules_cc` - rules for building C/C++ dependencies
* `protobuf` - required by the upgrade of `rules_cc`
* `bazel_skylib` - stdlib like Bazel rules

Note: This regresses our support for Apple x86_64 because I didn't have
a machine I could use to rebuild the `cargo-bazel` binary. But AFAIK
everyone using a Macbook now has an ARM based machine so this shouldn't
be an issue.

### Motivation

Unblocks our upgrade to Rust 1.86.0 and Edition 2024

### Checklist

- [ ] This PR has adequate test coverage / QA involvement has been duly
considered. ([trigger-ci for additional test/nightly
runs](https://trigger-ci.dev.materialize.com/))
- [ ] This PR has an associated up-to-date [design
doc](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/README.md),
is a design doc
([template](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/00000000_template.md)),
or is sufficiently small to not require a design.
  <!-- Reference the design in the description. -->
- [ ] If this PR evolves [an existing `$T ⇔ Proto$T`
mapping](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/command-and-response-binary-encoding.md)
(possibly in a backwards-incompatible way), then it is tagged with a
`T-proto` label.
- [ ] If this PR will require changes to cloud orchestration or tests,
there is a companion cloud PR to account for those changes that is
tagged with the release-blocker label
([example](MaterializeInc/cloud#5021)).
<!-- Ask in #team-cloud on Slack if you need help preparing the cloud
PR. -->
- [ ] If this PR includes major [user-facing behavior
changes](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/guide-changes.md#what-changes-require-a-release-note),
I have pinged the relevant PM to schedule a changelog post.

Author: ParkMyCar

Cargo.toml

@@ -242,6 +242,8 @@ exclude = [
     # All WASM crates are split into their own workspace to avoid needles cache
     #  invalidations for the core Mz crates.
     "misc/wasm/*",
+    # Ignore any Rust dependencies that python packages might pull in.
+    "misc/python/venv/*",
 ]
 
 # Use Cargo's new feature resolver, which can handle target-specific features.

WORKSPACE

@@ -31,9 +31,9 @@ included by calling a `*_dependencies()` macro.
 #
 # Note: In an ideal world the two rule sets would be combined into one.
 
-BAZEL_SKYLIB_VERSION = "1.6.1"
+BAZEL_SKYLIB_VERSION = "1.7.1"
 
-BAZEL_SKYLIB_INTEGRITY = "sha256-nziIakBUjG6WwQa3UvJCEw7hGqoGila6flb0UR8z5PI="
+BAZEL_SKYLIB_INTEGRITY = "sha256-vCg8381SalLDIBJ5zaS8KYZS76iYsQtNsIN9xRZSdW8="
 
 maybe(
     http_archive,
@@ -65,15 +65,22 @@ aspect_bazel_lib_dependencies()
 # Register bazel-lib toolchains
 aspect_bazel_lib_register_toolchains()
 
+# C Repositories
+#
+# Loads all of the C dependencies that we rely on.
+load("//misc/bazel/c_deps:repositories.bzl", "c_repositories")
+
+c_repositories()
+
 # `rules_cc`
 #
 # Rules for building C/C++ projects. These are slowly being upstreamed into the
 # Bazel source tree, but some projects (e.g. protobuf) still depend on this
 # rule set.
 
-RULES_CC_VERSION = "0.0.9"
+RULES_CC_VERSION = "0.1.1"
 
-RULES_CC_INTEGRITY = "sha256-IDeHW5pEVtzkp50RKorohbvEqtlo5lh9ym5k86CQDN8="
+RULES_CC_INTEGRITY = "sha256-cS13hosxUt1hjE1k+q3e/MWWX5D13m5t0dXdzQvoLUI="
 
 maybe(
     http_archive,
@@ -300,29 +307,22 @@ load("//misc/bazel/c_deps:extra_setup.bzl", "protoc_setup")
 
 protoc_setup()
 
-# C Repositories
-#
-# Loads all of the C dependencies that we rely on.
-load("//misc/bazel/c_deps:repositories.bzl", "c_repositories")
-
-c_repositories()
-
 # `rules_rust`
 #
 # Rules for building Rust crates, and several convienence macros for building all transitive
 # dependencies.
 
-RULES_RUST_VERSION = "0.54.2"
+RULES_RUST_VERSION = "0.59.3"
 
-RULES_RUST_INTEGRITY = "sha256-6hXbBNIbd6EvuBc/B5RMmAEhxLy5cs8EGuHpjqwjcz4="
+RULES_RUST_INTEGRITY = "sha256-pPPz9Yewxoqs6ZhcQAaI8AeFCy/S/ipQTEkTbZ3syz4="
 
 maybe(
     http_archive,
     name = "rules_rust",
     integrity = RULES_RUST_INTEGRITY,
-    strip_prefix = "rules_rust-v{0}".format(RULES_RUST_VERSION),
+    strip_prefix = "rules_rust-mz-{0}".format(RULES_RUST_VERSION),
     urls = [
-        "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/rules_rust-v{0}.tar.zst".format(RULES_RUST_VERSION),
+        "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/rules_rust-mz-{0}.tar.zst".format(RULES_RUST_VERSION),
     ],
 )
 
@@ -419,7 +419,17 @@ rust_toolchains(
 # Rules and Toolchains for running [`bindgen`](https://github.com/rust-lang/rust-bindgen)
 # a tool for generating Rust FFI bindings to C.
 
-load("@rules_rust//bindgen:repositories.bzl", "rust_bindgen_dependencies")
+maybe(
+    http_archive,
+    name = "rules_rust_bindgen",
+    integrity = RULES_RUST_INTEGRITY,
+    strip_prefix = "rules_rust-mz-{0}/extensions/bindgen".format(RULES_RUST_VERSION),
+    urls = [
+        "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/rules_rust-mz-{0}.tar.zst".format(RULES_RUST_VERSION),
+    ],
+)
+
+load("@rules_rust_bindgen//:repositories.bzl", "rust_bindgen_dependencies")
 
 rust_bindgen_dependencies()
 
@@ -461,6 +471,7 @@ crates_repository(
             # Note: The below targets are from the additive build file.
             additive_build_file = "@//misc/bazel/c_deps:rust-sys/BUILD.rocksdb.bazel",
             compile_data = [":out_dir"],
+            compile_data_glob_excludes = ["rocksdb/**"],
             gen_build_script = False,
             rustc_env = {
                 "OUT_DIR": "$(execpath :out_dir)",
@@ -564,16 +575,14 @@ crates_repository(
     cargo_config = "//:.cargo/config.toml",
     cargo_lockfile = "//:Cargo.lock",
     generator_sha256s = {
-        "aarch64-apple-darwin": "8204746334a17823bd6a54ce2c3821b0bdca96576700d568e2ca2bd8224dc0ea",
-        "x86_64-apple-darwin": "2ee14b230d32c05415852b7a388b76e700c87c506459e5b31ced19d6c131b6d0",
-        "aarch64-unknown-linux-gnu": "3792feb084bd43b9a7a9cd75be86ee9910b46db59360d6b29c9cca2f8889a0aa",
-        "x86_64-unknown-linux-gnu": "2b9d07f34694f63f0cc704989ad6ec148ff8d126579832f4f4d88edea75875b2",
+        "aarch64-apple-darwin": "c38c9c0efc11fcf9c32b9e0f4f4849df7c823f207c7f5ba5f6ab1e0e2167693d",
+        "aarch64-unknown-linux-gnu": "5bdc9a10ec5f17f5140a81ce7cb0c0ce6e82d4d862d3ce3a301ea23f72f20630",
+        "x86_64-unknown-linux-gnu": "abcd8212d64ea4c0f5e856af663c05ebeb2800a02c251f6eb62061f4e8ca1735",
     },
     generator_urls = {
-        "aarch64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/cargo-bazel-aarch64-apple-darwin".format(RULES_RUST_VERSION),
-        "x86_64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/cargo-bazel-x86_64-apple-darwin".format(RULES_RUST_VERSION),
-        "aarch64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/cargo-bazel-aarch64-unknown-linux-gnu".format(RULES_RUST_VERSION),
-        "x86_64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/cargo-bazel-x86_64-unknown-linux-gnu".format(RULES_RUST_VERSION),
+        "aarch64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/cargo-bazel-aarch64-apple-darwin".format(RULES_RUST_VERSION),
+        "aarch64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/cargo-bazel-aarch64-unknown-linux-gnu".format(RULES_RUST_VERSION),
+        "x86_64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/cargo-bazel-x86_64-unknown-linux-gnu".format(RULES_RUST_VERSION),
     },
     # When `isolated` is true, Bazel will create a new `$CARGO_HOME`, i.e. it
     # won't use `~/.cargo`, when re-pinning. This is nice but not totally

misc/bazel/c_deps/BUILD.bazel

@@ -35,7 +35,7 @@ build_test(
         "@jemalloc//:jemalloc",
         "@lz4//:lz4",
         "@openssl//:openssl",
-        "@protobuf//:protoc",
+        "@com_google_protobuf//:protoc",
         "@zlib//:zlib",
         "@zstd//:zstd",
     ],

misc/bazel/c_deps/extra_setup.bzl

@@ -23,7 +23,7 @@ def protoc_setup():
     they're specific to building `protoc` we split them out to reduce noise in
     that relatively crowded file.
 
-    Note: We could use "@protobuf//:protobuf_deps.bzl", but that pulls in
+    Note: We could use "@com_google_protobuf//:protobuf_deps.bzl", but that pulls in
     unneccessary toolchains that we don't need, like Java, so we manually
     specify dependencies here.
     """

misc/bazel/c_deps/repositories.bzl

@@ -81,11 +81,11 @@ def c_repositories():
         ],
     )
 
-    PROTOC_VERSION = "26.1"
-    PROTOC_INTEGRITY = "sha256-T8X/Gywzn7hs06JfC1MRR4qwgeZa0ljGeJNZzYTUIfg="
+    PROTOC_VERSION = "27.0"
+    PROTOC_INTEGRITY = "sha256-2iiL8dqmwE0DqQUXgcqlKs65FjWGv/mqbPsS9puTlao="
     maybe(
         http_archive,
-        name = "protobuf",
+        name = "com_google_protobuf",
         integrity = PROTOC_INTEGRITY,
         strip_prefix = "protobuf-{}".format(PROTOC_VERSION),
         urls = [

misc/bazel/c_deps/rust-sys/BUILD.protobuf-native.bazel

@@ -37,8 +37,8 @@ cc_library(
         ":lib-bridge/include",
         "@com_google_absl//absl/strings",
         "@crates_io__cxx-1.0.122//:cxx_cc",
-        "@protobuf//src/google/protobuf/compiler:code_generator",
-        "@protobuf//src/google/protobuf/compiler:importer",
+        "@com_google_protobuf//src/google/protobuf/compiler:code_generator",
+        "@com_google_protobuf//src/google/protobuf/compiler:importer",
     ],
 )
 

misc/bazel/c_deps/rust-sys/BUILD.rocksdb.bazel

@@ -18,7 +18,7 @@
 load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
 load("@bazel_skylib//rules:select_file.bzl", "select_file")
 load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake")
-load("@rules_rust//bindgen:defs.bzl", "rust_bindgen")
+load("@rules_rust_bindgen//:defs.bzl", "rust_bindgen")
 
 # Derived from <https://github.com/rust-rocksdb/rust-rocksdb/blob/7f9cba4a819e76d8022733b4c82509aec6056938/librocksdb-sys/build.rs>
 

misc/bazel/rust_deps/cxxbridge-cmd/Cargo.cxxbridge-cmd.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "b2b75eaba9abbc55071dcf7f2ded98069da446c19b24e7ac327c70bc8edf867e",
+  "checksum": "c17ea32c7611b5b8f4d91ef5bb280e78b142db085fdcc4ec608d3eb5303562a5",
   "crates": {
     "anstyle 1.0.4": {
       "name": "anstyle",
@@ -374,6 +374,9 @@
         "compile_data_glob": [
           "**"
         ],
+        "compile_data_glob_excludes": [
+          "**/*.rs"
+        ],
         "data_glob": [
           "**"
         ]
@@ -742,6 +745,9 @@
         "compile_data_glob": [
           "**"
         ],
+        "compile_data_glob_excludes": [
+          "**/*.rs"
+        ],
         "data_glob": [
           "**"
         ]
@@ -810,6 +816,9 @@
         "compile_data_glob": [
           "**"
         ],
+        "compile_data_glob_excludes": [
+          "**/*.rs"
+        ],
         "data_glob": [
           "**"
         ]
@@ -928,6 +937,9 @@
         "compile_data_glob": [
           "**"
         ],
+        "compile_data_glob_excludes": [
+          "**/*.rs"
+        ],
         "data_glob": [
           "**"
         ]
@@ -971,5 +983,6 @@
     "quote 1.0.33",
     "syn 2.0.38"
   ],
-  "direct_dev_deps": []
+  "direct_dev_deps": [],
+  "unused_patches": []
 }

misc/bazel/rust_deps/cxxbridge-cmd/include.BUILD.bazel

@@ -20,7 +20,7 @@ rust_binary(
     name = "cxxbridge-cmd",
     srcs = glob(["src/**/*.rs"]),
     aliases = aliases(),
-    data = [
+    compile_data = [
         "src/gen/include/cxx.h",
     ],
     edition = "2021",

misc/bazel/rust_deps/repositories.bzl

@@ -51,17 +51,15 @@ def rust_repositories():
             "x86_64-apple-darwin",
             "wasm32-unknown-unknown",
         ],
-        generator_urls = {
-            "aarch64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/v0.54.2/cargo-bazel-aarch64-apple-darwin",
-            "x86_64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/v0.54.2/cargo-bazel-x86_64-apple-darwin",
-            "aarch64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/v0.54.2/cargo-bazel-aarch64-unknown-linux-gnu",
-            "x86_64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/v0.54.2/cargo-bazel-x86_64-unknown-linux-gnu",
-        },
         generator_sha256s = {
-            "aarch64-apple-darwin": "8204746334a17823bd6a54ce2c3821b0bdca96576700d568e2ca2bd8224dc0ea",
-            "x86_64-apple-darwin": "2ee14b230d32c05415852b7a388b76e700c87c506459e5b31ced19d6c131b6d0",
-            "aarch64-unknown-linux-gnu": "3792feb084bd43b9a7a9cd75be86ee9910b46db59360d6b29c9cca2f8889a0aa",
-            "x86_64-unknown-linux-gnu": "2b9d07f34694f63f0cc704989ad6ec148ff8d126579832f4f4d88edea75875b2",
+            "aarch64-apple-darwin": "c38c9c0efc11fcf9c32b9e0f4f4849df7c823f207c7f5ba5f6ab1e0e2167693d",
+            "aarch64-unknown-linux-gnu": "5bdc9a10ec5f17f5140a81ce7cb0c0ce6e82d4d862d3ce3a301ea23f72f20630",
+            "x86_64-unknown-linux-gnu": "abcd8212d64ea4c0f5e856af663c05ebeb2800a02c251f6eb62061f4e8ca1735",
+        },
+        generator_urls = {
+            "aarch64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-0.59.3/cargo-bazel-aarch64-apple-darwin",
+            "aarch64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-0.59.3/cargo-bazel-aarch64-unknown-linux-gnu",
+            "x86_64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-0.59.3/cargo-bazel-x86_64-unknown-linux-gnu",
         },
         isolated = False,
         # Only used if developing rules_rust.

misc/bazel/toolchains/BUILD.bazel

@@ -27,7 +27,7 @@ Note: These registrations live here, and not in the `WORKSPACE` file or a
 See: <https://bazel.build/extending/toolchains>
 """
 
-load("@rules_rust//bindgen:defs.bzl", "rust_bindgen_toolchain")
+load("@rules_rust_bindgen//:defs.bzl", "rust_bindgen_toolchain")
 
 # Rust Bindgen Toolchains
 #
@@ -38,7 +38,7 @@ load("@rules_rust//bindgen:defs.bzl", "rust_bindgen_toolchain")
 # Darwin aarch64
 rust_bindgen_toolchain(
     name = "bindgen_toolchain_darwin__aarch64",
-    bindgen = "@rules_rust//bindgen/3rdparty:bindgen",
+    bindgen = "@rules_rust_bindgen//3rdparty:bindgen",
     clang = "@rust_bindgen__darwin_aarch64//:clang",
     libclang = "@rust_bindgen__darwin_aarch64//:libclang",
     libstdcxx = "@rust_bindgen__darwin_aarch64//:libc++",
@@ -50,14 +50,14 @@ toolchain(
         "@platforms//os:macos",
     ],
     toolchain = "bindgen_toolchain_darwin__aarch64",
-    toolchain_type = "@rules_rust//bindgen:toolchain_type",
+    toolchain_type = "@rules_rust_bindgen//:toolchain_type",
     visibility = ["//visibility:public"],
 )
 
 # Darwin x86_64
 rust_bindgen_toolchain(
     name = "bindgen_toolchain_darwin__x86_64",
-    bindgen = "@rules_rust//bindgen/3rdparty:bindgen",
+    bindgen = "@rules_rust_bindgen//3rdparty:bindgen",
     clang = "@rust_bindgen__darwin_x86_64//:clang",
     libclang = "@rust_bindgen__darwin_x86_64//:libclang",
     libstdcxx = "@rust_bindgen__darwin_x86_64//:libc++",
@@ -70,14 +70,14 @@ toolchain(
         "@platforms//cpu:x86_64",
     ],
     toolchain = "bindgen_toolchain_darwin__x86_64",
-    toolchain_type = "@rules_rust//bindgen:toolchain_type",
+    toolchain_type = "@rules_rust_bindgen//:toolchain_type",
     visibility = ["//visibility:public"],
 )
 
 # Linux aarch64
 rust_bindgen_toolchain(
     name = "bindgen_toolchain_linux__aarch64",
-    bindgen = "@rules_rust//bindgen/3rdparty:bindgen",
+    bindgen = "@rules_rust_bindgen//3rdparty:bindgen",
     clang = "@rust_bindgen__linux_aarch64//:clang",
     libclang = "@rust_bindgen__linux_aarch64//:libclang",
     libstdcxx = "@rust_bindgen__linux_aarch64//:libc++",
@@ -90,14 +90,14 @@ toolchain(
         "@platforms//cpu:aarch64",
     ],
     toolchain = "bindgen_toolchain_linux__aarch64",
-    toolchain_type = "@rules_rust//bindgen:toolchain_type",
+    toolchain_type = "@rules_rust_bindgen//:toolchain_type",
     visibility = ["//visibility:public"],
 )
 
 # Linux x86_64
 rust_bindgen_toolchain(
     name = "bindgen_toolchain_linux__x86_64",
-    bindgen = "@rules_rust//bindgen/3rdparty:bindgen",
+    bindgen = "@rules_rust_bindgen//3rdparty:bindgen",
     clang = "@rust_bindgen__linux_x86_64//:clang",
     libclang = "@rust_bindgen__linux_x86_64//:libclang",
     libstdcxx = "@rust_bindgen__linux_x86_64//:libc++",
@@ -110,6 +110,6 @@ toolchain(
         "@platforms//cpu:x86_64",
     ],
     toolchain = "bindgen_toolchain_linux__x86_64",
-    toolchain_type = "@rules_rust//bindgen:toolchain_type",
+    toolchain_type = "@rules_rust_bindgen//:toolchain_type",
     visibility = ["//visibility:public"],
 )

misc/python/materialize/cargo.py

@@ -148,7 +148,7 @@ def __init__(self, root: Path):
             self.crates[crate.name] = crate
         self.exclude: dict[str, Crate] = {}
         for path in workspace_config.get("exclude", []):
-            if path.endswith("*"):
+            if path.endswith("*") and (root / path.rstrip("*")).exists():
                 for item in (root / path.rstrip("*")).iterdir():
                     if item.is_dir() and (item / "Cargo.toml").exists():
                         crate = Crate(root, root / item)

src/build-tools/BUILD.bazel

@@ -28,8 +28,8 @@ rust_library(
         "bazel",
     ],
     data = [
-        "@protobuf//:protoc",
-        "@protobuf//:well_known_type_protos",
+        "@com_google_protobuf//:protoc",
+        "@com_google_protobuf//:well_known_type_protos",
     ],
     lint_config = ":lints",
     proc_macro_deps = [] + all_crate_deps(proc_macro = True),

src/build-tools/Cargo.toml

@@ -26,8 +26,8 @@ normal = ["workspace-hack"]
 features_override = ["default", "bazel"]
 extra_deps = ["@rules_rust//tools/runfiles"]
 data = [
-    "@protobuf//:protoc",
-    "@protobuf//:well_known_type_protos",
+    "@com_google_protobuf//:protoc",
+    "@com_google_protobuf//:well_known_type_protos",
 ]
 rustc_flags = ["--cfg=bazel"]
 

src/build-tools/src/lib.rs

@@ -49,7 +49,7 @@ pub fn protoc() -> PathBuf {
     cfg_if! {
         if #[cfg(bazel)] {
             let r = runfiles::Runfiles::create().unwrap();
-            r.rlocation("protobuf/protoc").expect("set by Bazel")
+            r.rlocation("com_google_protobuf/protoc").expect("set by Bazel")
         } else if #[cfg(feature = "protobuf-src")] {
             protobuf_src::protoc()
         } else {
@@ -70,7 +70,7 @@ pub fn protoc_include() -> PathBuf {
     cfg_if! {
         if #[cfg(bazel)] {
             let r = runfiles::Runfiles::create().unwrap();
-            r.rlocation("protobuf/src").expect("set by Bazel")
+            r.rlocation("com_google_protobuf/src").expect("set by Bazel")
         } else if #[cfg(feature = "protobuf-src")] {
             protobuf_src::include()
         } else {