debug

Author: ruyadorno

.github/workflows/benchmark.yaml

@@ -1,5 +1,5 @@
 name: Package Manager Benchmarks
-on: 
+on:
   push:
   workflow_dispatch:
     inputs:
@@ -33,15 +33,15 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        fixture: ${{ fromJson(inputs.fixtures || '["next", "astro", "vue", "svelte"]') }}
-        variation: ${{ fromJson(inputs.variations || '["cache", "cache+lockfile", "cache+node_modules", "cache+lockfile+node_modules", "clean", "lockfile", "lockfile+node_modules", "node_modules"]') }}
+        fixture: ${{ fromJson(inputs.fixtures || '["next"]') }}
+        variation: ${{ fromJson(inputs.variations || '["cache+lockfile+node_modules"]') }}
         include:
           - variation: "run"
             fixture: "run"
     env:
-      BENCH_INCLUDE: ${{ fromJson(inputs.binaries || '"npm,yarn,berry,pnpm,vlt,bun,deno,nx,turbo,node"') }}
+      BENCH_INCLUDE: ${{ fromJson(inputs.binaries || '"pnpm"') }}
       BENCH_WARMUP: ${{ inputs.warmup || '2' }}
-      BENCH_RUNS: ${{ inputs.runs || '10' }}
+      BENCH_RUNS: ${{ inputs.runs || '12' }}
     steps:
       - uses: actions/checkout@v4
       - name: Install Node
@@ -64,48 +64,3 @@ jobs:
           name: results-${{ matrix.fixture }}-${{ matrix.variation }}
           path: ./results/${{ matrix.fixture }}/${{ matrix.variation }}/
           retention-days: 7
-  process:
-    name: 'Process Results'
-    runs-on: ubuntu-latest
-    needs: [benchmark]
-    timeout-minutes: 5
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: '24'
-      - name: Download Results
-        uses: actions/download-artifact@v4
-        with:
-          path: results
-          pattern: results-*
-      - name: Process Results
-        run: |
-          bash ./scripts/process-results.sh
-      - name: Upload Processed Results
-        uses: actions/upload-artifact@v4
-        with:
-          name: charts
-          path: charts
-          retention-days: 7
-  deploy:
-    name: 'Deploy Results'
-    runs-on: ubuntu-latest
-    needs: [process]
-    permissions:
-      contents: write
-    # if: github.ref == 'refs/heads/main'
-    steps:
-      - uses: actions/checkout@v4
-      - name: Download Results
-        uses: actions/download-artifact@v4
-        with:
-          name: charts
-          path: charts/
-      - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: charts
-          force_orphan: true

scripts/package-count.sh

@@ -39,24 +39,32 @@ if [ -d "node_modules" ]; then
     | sort -u \
     | wc -l \
     | xargs
-  )
+  ) || true
 else
   # if there is no node_modules directory, the install failed
   # and we can't infer the package manager, so we'll skip this run
+  echo "Warning: node_modules directory does not exist"
   exit 0
 fi
 
-# If the package count is not set, set it to 0
-if [ -z "${BENCH_PACKAGE_COUNT:-}" ]; then
+# if we couldn't determine the package count, then just exit, no count file will be created
+if [ -z "${BENCH_PACKAGE_COUNT+x}" ] || [ "$BENCH_PACKAGE_COUNT" = "0" ]; then
   # if the package count couldn't be determined, the install likely failed
   # and we can't infer the package manager, so we'll skip this run
+  echo "Warning: Could not determine the package count"
   exit 0
 fi
 
 BENCH_PACKAGE_MANAGER=$(
   infer_package_manager
 )
 
+# if we couldn't determine the package manager, then just exit, no count file will be created
+if [ -z "${BENCH_PACKAGE_MANAGER+x}" ] || [ "$BENCH_PACKAGE_MANAGER" = "none" ]; then
+  echo "Warning: Could not determine the package manager"
+  exit 0
+fi
+
 # Create the results directory if it doesn't exist
 if [ -n "$BENCH_PACKAGE_MANAGER" ] && [ "$BENCH_PACKAGE_MANAGER" != "none" ]; then
   mkdir -p "$BENCH_PACKAGE_COUNT_FOLDER"

scripts/variations/cache+lockfile+node_modules.sh

@@ -8,6 +8,7 @@ source "$1/variations/common.sh"
 # When running a cache benchmark, we want to clean up only the node_modules
 # directory and the lockfiles between each run.
 hyperfine --ignore-failure \
+  --show-output \
   --time-unit=millisecond \
   --export-json="$BENCH_OUTPUT_FOLDER/benchmarks.json" \
   --warmup="$BENCH_WARMUP" \

scripts/variations/common.sh

@@ -73,5 +73,22 @@ mkdir -p "$BENCH_OUTPUT_FOLDER"
 # Function to collect package count into a package-count.json file
 collect_package_count() {
   ls -la "$BENCH_OUTPUT_FOLDER"
+
+  # Prints the output of each install
+  for pm in npm yarn berry pnpm vlt bun deno nx turbo node; do
+    if echo "$BENCH_INCLUDE" | grep -qw "$pm"; then
+      for i in {0..9}; do
+        echo "-- Reading output of $pm install $i ---"
+        log_file="$BENCH_OUTPUT_FOLDER/${pm}-output-$i.log"
+        if [ -f "$log_file" ]; then
+          echo "---"
+          cat "$log_file"
+          echo "---"
+        fi
+        echo "--------------------------------"
+      done
+    fi
+  done
+
   node "$BENCH_SCRIPTS/collect-package-count.js" "$BENCH_OUTPUT_FOLDER"
-}
+}