Valuecmp

.buildkite/hooks/post-checkout

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 CI_BUILD_START=$(date +%s)
 export CI_BUILD_START
 
@@ -8,14 +10,17 @@ source ci/env.sh
 # previous CI job
 #
 (
-  containers=$(docker ps -q)
-  if [[ $(hostname) != metrics-solana-com && -n $containers ]]; then
-    echo "+++ Killing stale docker containers"
-    docker ps
+  echo "+++ Killing stale docker containers"
+  while read -r line; do
+    read -r id image _ <<<"$line"
 
-    # shellcheck disable=SC2086 # Don't want to double quote $containers
-    docker kill $containers
-  fi
+    if [[ $image =~ "solanalabs/rust" ]]; then
+      if docker kill "$id" >/dev/null; then
+        echo "kill $id $image"
+      fi
+      continue
+    fi
+  done < <(docker ps | tail -n +2)
 )
 
 # Processes from previously aborted CI jobs seem to loiter, unclear why as one
@@ -37,6 +42,6 @@ source ci/env.sh
 # HACK: These are in our docker images, need to be removed from CARGO_HOME
 #  because we try to cache downloads across builds with CARGO_HOME
 # cargo lacks a facility for "system" tooling, always tries CARGO_HOME first
-cargo uninstall cargo-audit || true
-cargo uninstall svgbob_cli || true
-cargo uninstall mdbook || true
+cargo uninstall cargo-audit &>/dev/null || true
+cargo uninstall svgbob_cli &>/dev/null || true
+cargo uninstall mdbook &>/dev/null || true

.buildkite/hooks/post-command

@@ -1,11 +1,26 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+set -e
+
+source ci/upload-ci-artifact.sh
+
+#shellcheck source=ci/common/shared-functions.sh
+source ci/common/shared-functions.sh
 
 #
 # Add job_stats data point
 #
 if [[ -z $CI_BUILD_START ]]; then
   echo Error: CI_BUILD_START empty
 else
+  # make sure console outputs are uploaded as soon as possible, if any
+  (
+    shopt -s nullglob
+    for console_log in ./intercepted-console-*; do
+      gzip -f "$console_log"
+      upload-ci-artifact "$console_log.gz"
+    done
+  )
+
   CI_BUILD_DURATION=$(( $(date +%s) - CI_BUILD_START + 1 ))
 
   CI_LABEL=${BUILDKITE_LABEL:-build label missing}
@@ -20,15 +35,6 @@ else
     SUCCESS=false
   fi
 
-  if [[ -n $BUILDKITE && -f "results.json" ]]; then
-    # prepare result file
-    awk '/{ "type": .* }/' results.json > sanitized-results.json
-
-    # upload to datadog
-    cargo2junit > results.xml < sanitized-results.json
-    datadog-ci junit upload --service solana results.xml
-  fi
-
   point_tags="pipeline=$BUILDKITE_PIPELINE_SLUG,job=$CI_LABEL,pr=$PR,success=$SUCCESS"
   point_tags="${point_tags// /\\ }"  # Escape spaces
 

.buildkite/hooks/pre-command

@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-eval "$(ejson2env .buildkite/env/secrets.ejson)"
-
 # Ensure the pattern "+++ ..." never occurs when |set -x| is set, as buildkite
 # interprets this as the start of a log group.
 # Ref: https://buildkite.com/docs/pipelines/managing-log-output
@@ -12,10 +10,29 @@ export PS4="++"
 # Restore target/ from the previous CI build on this machine
 #
 eval "$(ci/channel-info.sh)"
-eval "$(ci/sbf-tools-info.sh)"
+export EDGE_CHANNEL
+export BETA_CHANNEL
+export BETA_CHANNEL_LATEST_TAG
+export STABLE_CHANNEL
+export STABLE_CHANNEL_LATEST_TAG
+export CHANNEL
+export CHANNEL_LATEST_TAG
+
+eval "$(ci/platform-tools-info.sh)"
 source "ci/rust-version.sh"
 HOST_RUST_VERSION="$rust_stable"
 pattern='^[0-9]+\.[0-9]+\.[0-9]+$'
 if [[ ${HOST_RUST_VERSION} =~ ${pattern} ]]; then
     HOST_RUST_VERSION="${rust_stable%.*}"
-fi
+fi
+
+export SBF_TOOLS_VERSION
+
+SCCACHE_KEY_PREFIX="${rust_stable}_${rust_nightly}_${SBF_TOOLS_VERSION}"
+export SCCACHE_KEY_PREFIX
+
+SCCACHE_S3_KEY_PREFIX="$SCCACHE_KEY_PREFIX"
+export SCCACHE_S3_KEY_PREFIX
+
+SCCACHE_GCS_KEY_PREFIX="$SCCACHE_KEY_PREFIX"
+export SCCACHE_GCS_KEY_PREFIX

.buildkite/scripts/build-bench.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/common.sh
+source "$here"/common.sh
+
+agent="${1-solana}"
+
+build_steps() {
+  cat <<EOF
+{
+  "name": "$1",
+  "command": "$2",
+  "timeout_in_minutes": 30,
+  "agent": "$agent",
+  "retry": 3
+}
+EOF
+}
+
+# shellcheck disable=SC2016
+group "bench" \
+  "$(build_steps "bench-part-1" "ci/bench/part1.sh")" \
+  "$(build_steps "bench-part-2" "ci/bench/part2.sh")"

.buildkite/scripts/build-bench.test.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/func-assert-eq.sh
+source "$here"/func-assert-eq.sh
+
+want=$(
+  cat <<'EOF'
+  - group: "bench"
+    steps:
+      - name: "bench-part-1"
+        command: "ci/bench/part1.sh"
+        timeout_in_minutes: 30
+        agents:
+          queue: "solana"
+        retry:
+          automatic:
+            - limit: 3
+      - name: "bench-part-2"
+        command: "ci/bench/part2.sh"
+        timeout_in_minutes: 30
+        agents:
+          queue: "solana"
+        retry:
+          automatic:
+            - limit: 3
+EOF
+)
+
+# shellcheck source=.buildkite/scripts/build-bench.sh
+got=$(source "$here"/build-bench.sh)
+
+assert_eq "test build bench steps" "$want" "$got"

.buildkite/scripts/build-downstream-projects.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/common.sh
+source "$here"/common.sh
+
+agent="${1-solana}"
+
+group "downstream projects"
+#  '{ "name": "spl", "command": "./ci/downstream-projects/run-spl.sh", "timeout_in_minutes": 30, "agent": "'"$agent"'" }' \
+#  '{ "name": "openbook-dex", "command": "./ci/downstream-projects/run-openbook-dex.sh", "timeout_in_minutes": 30, "agent": "'"$agent"'" }' \

.buildkite/scripts/build-stable.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/common.sh
+source "$here"/common.sh
+
+agent="${1-solana}"
+
+partitions=$(
+  cat <<EOF
+{
+  "name": "partitions",
+  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-partition.sh",
+  "timeout_in_minutes": 30,
+  "agent": "$agent",
+  "parallelism": 3,
+  "retry": 3
+}
+EOF
+)
+
+local_cluster_partitions=$(
+  cat <<EOF
+{
+  "name": "local-cluster",
+  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-local-cluster-partially.sh",
+  "timeout_in_minutes": 30,
+  "agent": "$agent",
+  "parallelism": 5,
+  "retry": 3
+}
+EOF
+)
+
+localnet=$(
+  cat <<EOF
+{
+  "name": "localnet",
+  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-localnet.sh",
+  "timeout_in_minutes": 30,
+  "agent": "$agent"
+}
+EOF
+)
+
+# shellcheck disable=SC2016
+group "stable" "$partitions" "$local_cluster_partitions" "$localnet"

.buildkite/scripts/build-stable.test.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/func-assert-eq.sh
+source "$here"/func-assert-eq.sh
+
+want=$(
+  cat <<'EOF'
+  - group: "stable"
+    steps:
+      - name: "partitions"
+        command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/stable/run-partition.sh"
+        timeout_in_minutes: 30
+        agents:
+          queue: "solana"
+        parallelism: 3
+        retry:
+          automatic:
+            - limit: 3
+      - name: "localnet"
+        command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/stable/run-localnet.sh"
+        timeout_in_minutes: 30
+        agents:
+          queue: "solana"
+EOF
+)
+
+# shellcheck source=.buildkite/scripts/build-stable.sh
+got=$(source "$here"/build-stable.sh)
+
+assert_eq "test build stable steps" "$want" "$got"