Merge branch 'main' into rf_1203

Author: zhang2014

.github/actions/benchmark_cloud/action.yml

@@ -10,6 +10,9 @@ inputs:
   dataset:
     description: "hits/tpch"
     required: true
+  database:
+    description: "Database name for benchmark queries"
+    required: false
   source:
     description: "pr/release"
     required: true
@@ -52,7 +55,11 @@ runs:
           echo "database=load_test_${{ inputs.run_id }}" >> $GITHUB_OUTPUT
           echo "tries=1" >> $GITHUB_OUTPUT
         else
-          echo "database=clickbench" >> $GITHUB_OUTPUT
+          database="${{ inputs.database }}"
+          if [[ -z "$database" ]]; then
+            database="clickbench"
+          fi
+          echo "database=$database" >> $GITHUB_OUTPUT
           echo "tries=3" >> $GITHUB_OUTPUT
         fi
 
@@ -65,37 +72,24 @@ runs:
         BENCHMARK_VERSION: ${{ inputs.version }}
         BENCHMARK_DATABASE: ${{ steps.prepare.outputs.database }}
         BENCHMARK_TRIES: ${{ steps.prepare.outputs.tries }}
+        BENCHMARK_SOURCE: ${{ inputs.source }}
+        BENCHMARK_SOURCE_ID: ${{ inputs.source_id }}
+        BENCHMARK_SHA: ${{ inputs.sha }}
         CLOUD_USER: ${{ inputs.cloud_user }}
         CLOUD_PASSWORD: ${{ inputs.cloud_password }}
         CLOUD_GATEWAY: ${{ inputs.cloud_gateway }}
         CLOUD_WAREHOUSE: benchmark-${{ inputs.run_id }}
       shell: bash
       run: |
-        ./benchmark_cloud.sh
-
-    - name: Prepare Metadata
-      working-directory: benchmark/clickbench
-      shell: bash
-      run: |
-        case ${{ inputs.source }} in
-          pr)
-            yq -i '.system = "Databend(PR#${{ inputs.source_id }})"' result.json
-            ;;
-          release)
-            yq -i '.system = "Databend(Release@${{ inputs.source_id }})"' result.json
-            ;;
-          *)
-            echo "Unsupported benchmark source: ${{ inputs.source }}"
-            exit 1
-        esac
-        yq -i '.comment = "commit: ${{ inputs.sha }}"' result.json
-        mv result.json result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
+        python3 benchmark_cloud.py
 
     - name: Upload artifact
       uses: actions/upload-artifact@v4
       with:
         name: benchmark-${{ inputs.dataset }}-${{ inputs.size }}
-        path: benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
+        path: |
+          benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
+          benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}-*.ndjson
 
     - name: Remove warehouse
       if: always()

.github/workflows/reuse.benchmark.yml

@@ -119,6 +119,7 @@ jobs:
           cloud_user: ${{ secrets.BENCHMARK_CLOUD_USER }}
           cloud_password: ${{ secrets.BENCHMARK_CLOUD_PASSWORD }}
           cloud_gateway: ${{ secrets.BENCHMARK_CLOUD_GATEWAY }}
+          database: load
       - name: clean
         if: always()
         continue-on-error: true
@@ -136,10 +137,11 @@ jobs:
     strategy:
       matrix:
         include:
-          - { dataset: hits, size: Small }
-          - { dataset: hits, size: Large }
-          - { dataset: tpch, size: Small }
-          - { dataset: tpch, size: Large }
+          - { dataset: hits, size: Small, database: clickbench, timeout: 10 }
+          - { dataset: hits, size: Large, database: clickbench, timeout: 10 }
+          - { dataset: tpch, size: Small, database: clickbench, timeout: 20 }
+          - { dataset: tpch, size: Large, database: clickbench, timeout: 20 }
+          - { dataset: tpch1000, size: Large, database: tpch_1000, timeout: 60 }
       fail-fast: true
       max-parallel: 1
     steps:
@@ -154,7 +156,7 @@ jobs:
       - uses: ./.github/actions/setup_bendsql
       - uses: ./.github/actions/benchmark_cloud
         if: inputs.target == 'cloud' || inputs.target == 'all'
-        timeout-minutes: 20
+        timeout-minutes: ${{ matrix.timeout }}
         id: benchmark
         with:
           sha: ${{ inputs.sha }}
@@ -167,6 +169,7 @@ jobs:
           cloud_user: ${{ secrets.BENCHMARK_CLOUD_USER }}
           cloud_password: ${{ secrets.BENCHMARK_CLOUD_PASSWORD }}
           cloud_gateway: ${{ secrets.BENCHMARK_CLOUD_GATEWAY }}
+          database: ${{ matrix.database }}
       - name: clean
         if: always()
         continue-on-error: true
@@ -191,11 +194,18 @@ jobs:
           pattern: benchmark-*
           merge-multiple: true
       - name: Get Report Prefix
+        working-directory: benchmark/clickbench
         run: |
-          for result in benchmark/clickbench/results/*.json; do
+          shopt -s nullglob
+          for result in results/*.json; do
             dataset=$(echo $result | sed -E 's/.*result-(\w+)-.*\.json/\1/')
-            mkdir -p benchmark/clickbench/results/${dataset}/
-            mv $result benchmark/clickbench/results/${dataset}/$(basename $result)
+            mkdir -p results/${dataset}/
+            mv $result results/${dataset}/$(basename $result)
+          done
+          for ndjson in results/*.ndjson; do
+            dataset=$(echo $ndjson | sed -E 's/.*result-(\w+)-.*\.ndjson/\1/')
+            mkdir -p results/${dataset}/
+            mv $ndjson results/${dataset}/$(basename $ndjson)
           done
           echo "REPORT_S3_PREFIX=s3://benchmark/clickbench/pr/${{ inputs.source_id }}/${{ inputs.run_id }}" >> $GITHUB_ENV
       - name: Upload PR clickbench result to R2
@@ -208,10 +218,12 @@ jobs:
         working-directory: benchmark/clickbench
         run: |
           echo -e "## ClickBench Report\n" > /tmp/body
+          shopt -s nullglob
           for p in results/*; do
+            [ -d "$p" ] || continue
             dataset=$(basename $p)
             aws s3 sync results/$dataset/ ${REPORT_S3_PREFIX}/ --include "*.json" --no-progress --checksum-algorithm=CRC32
-            aws s3 sync "s3://benchmark/clickbench/release/${dataset}/latest/" ./results/${dataset}/ --exclude "*" --include "*.json"
+            aws s3 sync "s3://benchmark/clickbench/release/${dataset}/latest/" ./results/${dataset}/ --exclude "*" --include "*.json" || true
             ./update_results.py --dataset $dataset --pr ${{ inputs.source_id }}
             aws s3 cp ./results/${dataset}.html ${REPORT_S3_PREFIX}/${dataset}.html --no-progress --checksum-algorithm=CRC32
             echo "* **${dataset}**: https://benchmark.databend.com/clickbench/pr/${{ inputs.source_id }}/${{ inputs.run_id }}/${dataset}.html" >> /tmp/body
@@ -233,6 +245,7 @@ jobs:
           - "tpch"
           - "hits"
           - "load"
+          - "tpch1000"
           # - "internal"
     steps:
       - uses: actions/checkout@v4
@@ -245,6 +258,20 @@ jobs:
           path: benchmark/clickbench/results
           pattern: benchmark-${{ matrix.dataset }}-*
           merge-multiple: true
+      - name: Prepare results directory
+        working-directory: benchmark/clickbench
+        run: |
+          shopt -s nullglob
+          for result in results/*.json; do
+            dataset=$(echo $result | sed -E 's/.*result-(\w+)-.*\.json/\1/')
+            mkdir -p results/${dataset}/
+            mv $result results/${dataset}/$(basename $result)
+          done
+          for ndjson in results/*.ndjson; do
+            dataset=$(echo $ndjson | sed -E 's/.*result-(\w+)-.*\.ndjson/\1/')
+            mkdir -p results/${dataset}/
+            mv $ndjson results/${dataset}/$(basename $ndjson)
+          done
       - name: Generate report and upload to R2
         working-directory: benchmark/clickbench
         env:
@@ -253,12 +280,6 @@ jobs:
           AWS_DEFAULT_REGION: auto
           AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
         run: |
-          for result in results/*.json; do
-            dataset=$(echo $result | sed -E 's/.*result-(\w+)-.*\.json/\1/')
-            mkdir -p results/${dataset}/
-            mv $result results/${dataset}/$(basename $result)
-          done
-
           aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date --date='-1 month' -u +%Y)/$(date --date='-1 month' -u +%m)/ ./results/${{ matrix.dataset }}/
           aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/ ./results/${{ matrix.dataset }}/
           ./update_results.py --dataset ${{ matrix.dataset }} --release ${{ inputs.source_id }}
@@ -271,3 +292,20 @@ jobs:
           done
 
           aws s3 cp ./results/${{ matrix.dataset }}.html s3://benchmark/clickbench/release/${{ matrix.dataset }}.html --no-progress --checksum-algorithm=CRC32
+      - name: Upload NDJSON archives to R2
+        working-directory: benchmark/clickbench
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
+          DATASET: ${{ matrix.dataset }}
+        run: |
+          shopt -s nullglob
+          MONTH=$(date -u +%Y-%m)
+          ARCHIVE_PREFIX="s3://benchmark/results/${DATASET}/${MONTH}/"
+          IMPORT_PREFIX="s3://benchmark/results/import/"
+          for file in ./results/${DATASET}/*.ndjson; do
+            aws s3 cp $file "${ARCHIVE_PREFIX}$(basename $file)" --no-progress --checksum-algorithm=CRC32
+            aws s3 cp $file "${IMPORT_PREFIX}$(basename $file)" --no-progress --checksum-algorithm=CRC32
+          done

Cargo.lock

@@ -487,6 +487,7 @@ simple_hll = { version = "0.0.4", features = ["serde_borsh"] }
 simsearch = "0.2"
 siphasher = "0.3"
 sled = { version = "0.34", default-features = false }
+smallvec = "1.13.2"
 snailquote = "0.3.1"
 snap = "1"
 socket2 = "0.5.3"
@@ -548,6 +549,7 @@ cargo_metadata = "0.19"
 fast-float2 = "0.2.3"
 gix = "0.71.0"
 indent = "0.1.1"
+inventory = "0.3.15"
 logos = "0.12.1"
 nom = "8.0.0"
 nom-rule = "0.5.1"