rewrite benchmark suite

Rewrites the benchmark suite to include a full matrix of different
variations, among 4 different fixtures and 7 different scenarios
along with fixing the graph render and url deploy.

Fixes: vltpkg/statusboard#197
Fixes: vltpkg/statusboard#9
Fixes: vltpkg/statusboard#28

Author: ruyadorno

.editorconfig

@@ -0,0 +1,44 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# All files
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+# Shell scripts
+[*.sh]
+indent_style = space
+indent_size = 2
+
+# JavaScript/TypeScript files
+[*.{js,mjs,ts}]
+indent_style = space
+indent_size = 2
+
+# JSON files
+[*.json]
+indent_style = space
+indent_size = 2
+
+# CSS files
+[*.css]
+indent_style = space
+indent_size = 2
+
+# HTML files
+[*.html]
+indent_style = space
+indent_size = 2
+# Markdown files
+[*.md]
+trim_trailing_whitespace = false
+
+# Package.json files
+[package.json]
+indent_style = space
+indent_size = 2 

.github/workflows/benchmark.yaml

@@ -1,171 +1,112 @@
 name: Package Manager Benchmarks
-on: 
+on:
   push:
   workflow_dispatch:
+    inputs:
+      fixtures:
+        description: 'The fixture to run the benchmarks on'
+        default: '["next", "astro", "svelte", "vue"]'
+      variations:
+        description: 'The benchmark variations to run'
+        default: '["cache", "cache+lockfile", "cache+node_modules", "cache+lockfile+node_modules", "clean", "lockfile", "lockfile+node_modules", "node_modules"]'
+      binaries:
+        description: 'The binaries to run the benchmarks on'
+        default: '"npm,yarn,berry,pnpm,vlt,bun,deno,nx,turbo,node"'
+      warmup:
+        description: 'The number of warmup runs on each benchmark'
+        default: '2'
+      runs:
+        description: 'The number of runs on each benchmark'
+        default: '10'
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "5 23 * * *"
 
 # Prevent multiple runs from interfering with each other
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  setup: 
-    name: 'Setup'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v4
-      - name: Cache Bins
-        uses: useblacksmith/cache@v5
-        with:
-          path: bins
-          key: bins
-      - name: Install Node
-        uses: useblacksmith/setup-node@v5
-        with:
-          node-version: '22'
-      - name: Install & Setup Tools
-        run: |
-          bash ./scripts/setup.sh
-
   benchmark:
     name: 'Run Benchmarks'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [setup]
-    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
     strategy:
       matrix:
-        project: [next, astro, svelte, vue]
-        cache: [cold, warm]
+        fixture: ${{ fromJson(inputs.fixtures || '["next", "astro", "vue", "svelte"]') }}
+        variation: ${{ fromJson(inputs.variations || '["cache", "cache+lockfile", "cache+node_modules", "cache+lockfile+node_modules", "clean", "lockfile", "lockfile+node_modules", "node_modules"]') }}
         include:
-          - project: next
-            cache: cold
-          - project: astro
-            cache: cold
-          - project: svelte
-            cache: cold
-          - project: vue
-            cache: cold
-          - project: next
-            cache: warm
-          - project: astro
-            cache: warm
-          - project: svelte
-            cache: warm
-          - project: vue
-            cache: warm
+          - variation: "run"
+            fixture: "run"
+    env:
+      BENCH_INCLUDE: ${{ fromJson(inputs.binaries || '"npm,yarn,berry,pnpm,vlt,bun,deno,nx,turbo,node"') }}
+      BENCH_WARMUP: ${{ inputs.warmup || '2' }}
+      BENCH_RUNS: ${{ inputs.runs || '10' }}
     steps:
       - uses: actions/checkout@v4
       - name: Install Node
-        uses: useblacksmith/setup-node@v5
+        uses: actions/setup-node@v4
         with:
-          node-version: '22'
-      - name: Install System Dependencies
+          node-version: '24'
+      - name: Install & Setup Tools
         run: |
-          sudo apt-get update
-          sudo apt-get install -y jq hyperfine
-      - name: Restore Bins
-        uses: useblacksmith/cache/restore@v5
-        with:
-          path: bins
-          key: bins
-      - name: Run Project Benchmarks
+          bash ./scripts/setup.sh
+      - name: Run Benchmarks variations
         run: |
-          if [ "${{ matrix.cache }}" = "warm" ]; then
-            bash ./scripts/install-warm.sh ${{ matrix.project }}
+          if [ "${{ matrix.variation }}" = "run" ]; then
+            bash ./scripts/benchmark.sh run run
           else
-            bash ./scripts/install.sh ${{ matrix.project }}
+            bash ./scripts/benchmark.sh ${{ matrix.fixture }} ${{ matrix.variation }}
           fi
       - name: Upload Benchmark Results
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.project }}-${{ matrix.cache }}-results
-          path: ./results/${{ matrix.project }}/
-          retention-days: 7
-
-  task:
-    name: 'Benchmark Running Tasks'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [setup]
-    timeout-minutes: 15
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install Node
-        uses: useblacksmith/setup-node@v5
-        with:
-          node-version: '22'
-      - name: Install System Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y jq hyperfine
-      - name: Restore Bins
-        uses: useblacksmith/cache/restore@v5
-        with:
-          path: bins
-          key: bins
-      - name: Run Task Execution Benchmarks
-        run: |
-          bash ./scripts/run.sh
-      - name: Upload Benchmark Results
-        uses: actions/upload-artifact@v4
-        with:
-          name: task-benchmark-results
-          path: ./results/run.json
+          name: results-${{ matrix.fixture }}-${{ matrix.variation }}
+          path: ./results/${{ matrix.fixture }}/${{ matrix.variation }}/
           retention-days: 7
-
   process:
     name: 'Process Results'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [benchmark, task]
+    runs-on: ubuntu-latest
+    needs: [benchmark]
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v4
       - name: Install Node
-        uses: useblacksmith/setup-node@v5
+        uses: actions/setup-node@v4
         with:
-          node-version: '22'
+          node-version: '24'
       - name: Download Results
         uses: actions/download-artifact@v4
         with:
           path: results
-          merge-multiple: true
+          pattern: results-*
       - name: Process Results
         run: |
           bash ./scripts/process-results.sh
       - name: Upload Processed Results
         uses: actions/upload-artifact@v4
         with:
-          name: processed-results
-          path: chart/results/
+          name: charts
+          path: charts
           retention-days: 7
-
   deploy:
     name: 'Deploy Results'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     needs: [process]
+    permissions:
+      contents: write
     if: github.ref == 'refs/heads/main'
     steps:
       - uses: actions/checkout@v4
       - name: Download Results
         uses: actions/download-artifact@v4
         with:
-          name: processed-results
-          path: chart/results/
+          name: charts
+          path: charts/
       - name: Deploy to GitHub Pages
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: chart/results
+          publish_dir: charts
           force_orphan: true
 
-  cleanup:
-    name: 'Cleanup'
-    needs: [deploy]
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    if: always()
-    steps:
-      - name: Delete old artifacts
-        run: |
-          gh api repos/${{ github.repository }}/actions/artifacts --paginate | jq -r '.artifacts[] | select(.expired) | .id' | xargs -I {} gh api repos/${{ github.repository }}/actions/artifacts/{} -X DELETE

LICENSE

@@ -0,0 +1,15 @@
+Copyright (c) vlt technology, Inc.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+   Subject to the terms and conditions of this license, each copyright holder and contributor hereby grants to those receiving rights under this license a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except for failure to satisfy the conditions of this license) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer this software, where such license applies only to those patent claims, already acquired or hereafter acquired, licensable by such copyright holder or contributor that are necessarily infringed by:
+
+(a) their Contribution(s) (the licensed copyrights of copyright holders and non-copyrightable additions of contributors, in source or binary form) alone; or
+(b) combination of their Contribution(s) with the work of authorship to which such Contribution(s) was added by such copyright holder or contributor, if, at the time the Contribution is added, such addition causes such combination to be necessarily infringed. The patent license shall not apply to any other combinations which include the Contribution.
+Except as expressly stated above, no rights or licenses from any copyright holder or contributor is granted under this license, whether expressly, by implication, estoppel or otherwise.
+
+DISCLAIMER
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.