Merge remote-tracking branch 'upstream/main' into source-table-migration

.github/CODEOWNERS

@@ -14,7 +14,7 @@
 /bin
 /bin/lint-versions                  @MaterializeInc/testing
 /ci                                 @MaterializeInc/testing
-/ci/test/lint-deps.toml             @danhhz @benesch
+/ci/test/lint-deps.toml
 /doc/user                           @MaterializeInc/docs
 /doc/developer/reference/compute    @MaterializeInc/cluster
 /doc/developer/reference/storage    @MaterializeInc/cluster
@@ -46,7 +46,7 @@
 /src/adapter/src/coord/introspection.rs @MaterializeInc/cluster
 # to track changes to feature flags
 /src/adapter/src/flags.rs           @MaterializeInc/testing
-/src/alloc                          @benesch
+/src/alloc
 /src/audit-log                      @MaterializeInc/adapter
 /src/avro                           @MaterializeInc/cluster
 /src/avro-derive                    @MaterializeInc/cluster
@@ -65,8 +65,8 @@
 /src/controller                     @MaterializeInc/cluster
 /src/controller-types               @MaterializeInc/cluster
 /src/durable-cache                  @MaterializeInc/adapter
-/src/dyncfg                         @danhhz
-/src/dyncfgs                        @danhhz
+/src/dyncfg
+/src/dyncfgs
 /src/environmentd                   @MaterializeInc/adapter
 /src/expr                           @MaterializeInc/cluster
 /src/expr-parser                    @MaterializeInc/cluster
@@ -78,15 +78,15 @@
 /src/kafka-util                     @MaterializeInc/cluster
 /src/lowertest                      @MaterializeInc/cluster
 /src/lowertest-derive               @MaterializeInc/cluster
-/src/metabase                       @benesch
+/src/metabase
 /src/mysql-util                     @MaterializeInc/cluster
 /src/mz                             @MaterializeInc/integrations
-/src/npm                            @benesch
+/src/npm
 /src/orchestrator                   @MaterializeInc/cloud
 /src/orchestrator-kubernetes        @MaterializeInc/cloud
 /src/orchestrator-process           @MaterializeInc/cloud
 /src/orchestrator-tracing           @MaterializeInc/cloud
-/src/ore                            @benesch
+/src/ore
 /src/persist                        @MaterializeInc/persist
 /src/persist-cli                    @MaterializeInc/persist
 /src/persist-client                 @MaterializeInc/persist
@@ -107,7 +107,7 @@
 /src/rocksdb                        @MaterializeInc/cluster
 /src/s3-datagen                     @MaterializeInc/cluster
 /src/secrets                        @MaterializeInc/cloud
-/src/segment                        @benesch
+/src/segment
 /src/service                        @MaterializeInc/cluster
 /src/sql                            @MaterializeInc/adapter
 /src/sql/src/session/vars.rs
@@ -132,7 +132,7 @@
 /src/testdrive                      @MaterializeInc/testing
 /src/timely-util                    @MaterializeInc/cluster
 /src/transform                      @MaterializeInc/cluster
-/src/txn-wal                        @aljoscha @danhhz @jkosh44
+/src/txn-wal                        @aljoscha
 /src/walkabout                      @MaterializeInc/adapter
 /src/workspace-hack                 @MaterializeInc/testing
 /test

LICENSE

@@ -13,7 +13,7 @@ Business Source License 1.1
 
 Licensor:                  Materialize, Inc.
 
-Licensed Work:             Materialize Version 20250128
+Licensed Work:             Materialize Version 20250129
                            The Licensed Work is © 2025 Materialize, Inc.
 
 Additional Use Grant:      Within a single installation of Materialize, you
@@ -32,7 +32,7 @@ Additional Use Grant:      Within a single installation of Materialize, you
                            whose definitions are controlled by such third
                            parties.
 
-Change Date:               January 28, 2029
+Change Date:               January 29, 2029
 
 Change License:            Apache License, Version 2.0
 

bin/ci-builder

@@ -188,6 +188,7 @@ case "$cmd" in
             --env COMMON_ANCESTOR_OVERRIDE
             --env CONFLUENT_CLOUD_DEVEX_KAFKA_PASSWORD
             --env CONFLUENT_CLOUD_DEVEX_KAFKA_USERNAME
+            --env GCP_SERVICE_ACCOUNT_JSON
             --env GITHUB_TOKEN
             --env GPG_KEY
             --env LAUNCHDARKLY_API_TOKEN

ci/builder/Dockerfile

@@ -326,7 +326,17 @@ RUN curl -fsSL https://github.com/deb-s3/deb-s3/releases/download/0.11.3/deb-s3-
 RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-$ARCH_GCC-2.17.2.zip" > awscli.zip \
     && unzip awscli.zip \
     && ./aws/install \
-    && rm -rf aws
+    && rm -rf aws awscli.zip
+
+# Install the gcloud CLI.
+
+RUN arch_gcloud=$(echo "$ARCH_GCC" | sed -e "s/aarch64/arm/" -e "s/amd64/x86_64/") \
+    && curl -fsSL "https://storage.googleapis.com/cloud-sdk-release/google-cloud-cli-507.0.0-linux-$arch_gcloud.tar.gz" > gcloud.tar.gz \
+    && tar -xzf gcloud.tar.gz -C /opt \
+    && rm gcloud.tar.gz \
+    && CLOUDSDK_CORE_DISABLE_PROMPTS=1 /opt/google-cloud-sdk/install.sh --path-update false \
+    && /opt/google-cloud-sdk/bin/gcloud config set disable_usage_reporting false \
+    && /opt/google-cloud-sdk/bin/gcloud components install gke-gcloud-auth-plugin
 
 # Install docs site dependencies. These are towards the end for the same reason
 # as the Python dependencies. These are only supported on x86_64 at the moment.
@@ -410,7 +420,7 @@ ENV TARGET_CC=$CC
 ENV TARGET_CXX=$CXX
 ENV TARGET_CXXSTDLIB=static=stdc++
 ENV TARGET_RANLIB=$RANLIB
-ENV PATH=/opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin:$PATH
+ENV PATH=/opt/google-cloud-sdk/bin:/opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin:$PATH
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-unknown-linux-gnu-cc
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-unknown-linux-gnu-cc
 ENV CARGO_TARGET_DIR=/mnt/build

ci/nightly/pipeline.template.yml

@@ -1260,13 +1260,30 @@ steps:
       depends_on: build-aarch64
       timeout_in_minutes: 1200
       concurrency: 1
-      concurrency_group: 'terraform'
+      concurrency_group: 'terraform-aws'
       agents:
         queue: linux-aarch64-small
       plugins:
         - ./ci/plugins/scratch-aws-access: ~
         - ./ci/plugins/mzcompose:
             composition: terraform
+            run: aws
+      branches: "main v*.* lts-v*"
+
+    - id: terraform-gcp
+      label: "Terraform + Helm Chart E2E on GCP"
+      artifact_paths: [test/terraform/aws/terraform.tfstate]
+      depends_on: build-aarch64
+      timeout_in_minutes: 1200
+      concurrency: 1
+      concurrency_group: 'terraform-gcp'
+      agents:
+        queue: linux-aarch64-small
+      plugins:
+        - ./ci/plugins/scratch-aws-access: ~
+        - ./ci/plugins/mzcompose:
+            composition: terraform
+            run: gcp
       branches: "main v*.* lts-v*"
 
   - group: "Output consistency"

ci/plugins/mzcompose/hooks/pre-exit

@@ -85,7 +85,7 @@ timeout 300 buildkite-agent artifact upload "$artifacts_str" || true
 bin/ci-builder run stable bin/ci-annotate-errors --test-cmd="$(cat test_cmd)" --test-desc="$(cat test_desc)" "${artifacts[@]}" > ci-annotate-errors.log || CI_ANNOTATE_ERRORS_RESULT=$?
 buildkite-agent artifact upload "ci-annotate-errors.log"
 
-if [ ! -s services.log ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] && [ "$BUILDKITE_LABEL" != "Mz E2E Test" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for DFR)" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for CTF)" ] && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on AWS" ]; then
+if [ ! -s services.log ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] && [ "$BUILDKITE_LABEL" != "Mz E2E Test" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for DFR)" ] && [ "$BUILDKITE_LABEL" != "Output consistency (version for CTF)" ] && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on AWS" ] && [ "$BUILDKITE_LABEL" != "Terraform + Helm Chart E2E on GCP" ]; then
     echo "+++ services.log is empty, failing"
     exit 1
 fi
@@ -145,3 +145,7 @@ ci_unimportant_heading ":docker: Cleaning up after mzcompose"
 run kill || true
 run rm --force -v
 run down --volumes
+
+ci_unimportant_heading "terraform: Destroying leftover state in case job was cancelled or timed out..."
+bin/ci-builder run stable terraform -chdir=test/terraform/aws destroy || true
+bin/ci-builder run stable terraform -chdir=test/terraform/gcp destroy || true

ci/test/lint-main/checks/check-mzcompose-files.sh

@@ -46,6 +46,7 @@ check_default_workflow_references_others() {
         -not -wholename "./test/ssh-connection/mzcompose.py" `# Handled differently` \
         -not -wholename "./test/scalability/mzcompose.py" `# Other workflows are for manual usage` \
         -not -wholename "./test/testdrive-old-kafka-src-syntax/mzcompose.py" `# Other workflow is run separately` \
+        -not -wholename "./test/terraform/mzcompose.py" `# Handled differently` \
     )
 
     for file in "${MZCOMPOSE_TEST_FILES[@]}"; do

src/adapter-types/Cargo.toml

@@ -15,7 +15,7 @@ mz-ore = { path = "../ore" }
 mz-repr = { path = "../repr" }
 mz-storage-types = { path = "../storage-types" }
 serde = "1.0.152"
-timely = "0.16.0"
+timely = "0.17.0"
 workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true }
 
 [package.metadata.cargo-udeps.ignore]

src/adapter/Cargo.toml

@@ -18,7 +18,7 @@ bytesize = "1.1.0"
 chrono = { version = "0.4.35", default-features = false, features = ["std"] }
 dec = "0.4.8"
 derivative = "2.2.0"
-differential-dataflow = "0.13.3"
+differential-dataflow = "0.13.4"
 enum-kinds = "0.5.1"
 fail = { version = "0.5.1", features = ["failpoints"] }
 futures = "0.3.25"
@@ -81,7 +81,7 @@ serde_plain = "1.0.1"
 sha2 = "0.10.6"
 smallvec = { version = "1.10.0", features = ["union"] }
 static_assertions = "1.1"
-timely = "0.16.0"
+timely = "0.17.0"
 tokio = { version = "1.38.0", features = ["rt", "time"] }
 tokio-postgres = { version = "0.7.8" }
 tracing = "0.1.37"