Merge pull request #28 from pytorch/run-benchmark-all-commits

Add a cron shell script to run all vLLM commits from its main branch

Author: huydhn

Diff for: .gitignore

@@ -1,3 +1,4 @@
 output/
 *.log
 *.xml
+vllm-benchmarks/commit

Diff for: vllm-benchmarks/README.md

@@ -11,6 +11,12 @@ take sometimes.
 
 ### vLLM benchmark on PyTorch infra
 
+* Run the benchmark on all vLLM main commits continuously
+
+```
+HF_TOKEN=<REDACTED> ./cron.sh
+```
+
 * Run the benchmark on the latest commit in a branch, i.e. `main`
 
 ```

Diff for: vllm-benchmarks/cron.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -eux
+
+pull_vllm() {
+  # I'm doing the checkout step here so that this script can be run without GHA
+  if [[ ! -d "vllm" ]]; then
+    git clone https://github.com/vllm-project/vllm.git
+  fi
+
+  pushd vllm
+  # Clean up any local changes to the benchmark suite
+  git checkout .buildkite/nightly-benchmarks/
+
+  git checkout main
+  git fetch origin && git pull origin main
+  popd
+}
+
+run() {
+  COMMIT=$1
+  HEAD_BRANCH=main
+
+  set +e
+  ./run.sh ${COMMIT}
+  set -eux
+
+  NOT_EXIST=0
+
+  S3_PATH="v3/vllm-project/vllm/${HEAD_BRANCH}/${COMMIT}/benchmark_results.json"
+  aws s3api head-object --bucket ossci-benchmarks --key ${S3_PATH} || NOT_EXIST=1
+
+  if [[ ${NOT_EXIST:-0} == "0" ]]; then
+    echo "${COMMIT}" > commit
+    echo "Mark ${COMMIT} as the latest commit that has been benchmarked on main"
+
+    S3_PATH="last-green-commits/vllm-project/vllm/${HEAD_BRANCH}/commit"
+    aws s3 cp commit "s3://ossci-benchmarks/${S3_PATH}"
+  fi
+}
+
+run_benchmarks() {
+  pushd vllm
+  HEAD_BRANCH=main
+  HEAD_SHA=$(git rev-parse --verify HEAD)
+  popd
+
+  rm commit || true
+  # Get the last green commit from S3
+  S3_PATH="last-green-commits/vllm-project/vllm/${HEAD_BRANCH}/commit"
+  aws s3 cp "s3://ossci-benchmarks/${S3_PATH}" .
+  LAST_GREEN_COMMIT=$(cat commit)
+
+  if [[ "${LAST_GREEN_COMMIT}" == "${HEAD_SHA}" ]]; then
+    echo "Skip ${HEAD_BRANCH}/${HEAD_SHA} because all older commits have already been benchmarked"
+  else
+    COMMITS=$(python get_commits.py --repo vllm --from-commit ${LAST_GREEN_COMMIT})
+    echo "${COMMITS}" | while IFS= read -r COMMIT ; do run ${COMMIT} ; done
+  fi
+}
+
+while :
+do
+  pull_vllm
+  run_benchmarks
+  sleep 300
+done
+
+popd

Diff for: vllm-benchmarks/get_commits.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+import os
+from typing import Any, List, Optional
+from git import Repo
+from argparse import Action, ArgumentParser, Namespace
+
+
+MAX_LOOKBACK = 100
+
+
+class ValidateDir(Action):
+    def __call__(
+        self,
+        parser: ArgumentParser,
+        namespace: Namespace,
+        values: Any,
+        option_string: Optional[str] = None,
+    ) -> None:
+        if os.path.isdir(values):
+            setattr(namespace, self.dest, values)
+            return
+
+        parser.error(f"{values} is not a valid directory")
+
+
+def parse_args() -> Any:
+    parser = ArgumentParser("Get the list of commits from a repo")
+    parser.add_argument(
+        "--repo",
+        type=str,
+        required=True,
+        action=ValidateDir,
+        help="the directory that the repo is checked out",
+    )
+    parser.add_argument(
+        "--from-commit",
+        type=str,
+        required=True,
+        help="gather all commits from this commit (exclusive)",
+    )
+    parser.add_argument(
+        "--to-commit",
+        type=str,
+        default="",
+        help="gather all commits to this commit (inclusive)",
+    )
+    parser.add_argument(
+        "--branch",
+        type=str,
+        default="main",
+        help="the target branch",
+    )
+
+    return parser.parse_args()
+
+
+def get_commits(
+    repo_dir: str, branch_name: str, from_commit: str, to_commit: str
+) -> List[str]:
+    commits = []
+    found_to_commit = True if to_commit == "" else False
+
+    repo = Repo(repo_dir)
+    # The commit is sorted where the latest one comes first
+    for index, commit in enumerate(repo.iter_commits(branch_name)):
+        if index > MAX_LOOKBACK:
+            break
+
+        if not found_to_commit and str(commit) == to_commit:
+            found_to_commit = True
+
+        if not found_to_commit:
+            continue
+
+        if str(commit) == from_commit:
+            break
+
+        commits.append(commit)
+
+    return commits
+
+
+def main() -> None:
+    args = parse_args()
+    for commit in reversed(
+        get_commits(
+            args.repo,
+            args.branch,
+            args.from_commit,
+            args.to_commit,
+        )
+    ):
+        print(commit)
+
+
+if __name__ == "__main__":
+    main()

Diff for: vllm-benchmarks/run.sh

@@ -24,7 +24,6 @@ setup_vllm() {
   fi
 
   pushd vllm
-
   # Clean up any local changes to the benchmark suite
   git checkout .buildkite/nightly-benchmarks/
 
@@ -97,7 +96,7 @@ cleanup
 setup_vllm
 
 pushd vllm
-export HEAD_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+export HEAD_BRANCH=main
 export HEAD_SHA=$(git rev-parse --verify HEAD)
 
 S3_PATH="v3/vllm-project/vllm/${HEAD_BRANCH}/${HEAD_SHA}/benchmark_results.json"

Diff for: vllm-benchmarks/upload_benchmark_results.py

@@ -71,7 +71,11 @@ def parse_args() -> Any:
 
 def get_git_metadata(vllm_dir: str) -> Tuple[str, str]:
     repo = Repo(vllm_dir)
-    return repo.active_branch.name, repo.head.object.hexsha
+    try:
+        return repo.active_branch.name, repo.head.object.hexsha
+    except TypeError:
+        # This is a detached HEAD, default the branch to main
+        return "main", repo.head.object.hexsha
 
 
 def get_benchmark_metadata(head_branch: str, head_sha: str) -> Dict[str, Any]: