chore(benchmarks): Add in optional warmup to benchmarks (#2359)

sydney-munro · gcf-owl-bot[bot] · web-flow · commit f88bcf837893 · 2024-01-23T13:45:27.000-08:00
* chore(benchmarks): Add in 15s warmup to benchmarks * make warmup configurable * fix comment that says 15s warmup * linter * Address PR comments * PR comments * refactor warmup into runW1R3 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
diff --git a/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java
@@ -17,10 +17,8 @@
 package com.google.cloud.storage.benchmarking;
 
 import com.google.api.core.ApiFuture;
-import com.google.api.core.ApiFutures;
 import com.google.api.core.ListenableFutureToApiFuture;
 import com.google.api.gax.retrying.RetrySettings;
-import com.google.api.gax.rpc.ApiExceptions;
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 import com.google.common.util.concurrent.ListenableFuture;
@@ -29,9 +27,8 @@
 import java.io.PrintWriter;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Random;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.regex.Pattern;
 import picocli.CommandLine;
@@ -83,13 +80,28 @@ public final class StorageSharedBenchmarkingCli implements Runnable {
       description = "Specify the path where the temporary directory should be located")
   String tempDirLocation;
 
+  @Option(
+      names = "-warmup",
+      description = "Number of seconds a W1R3 warmup will run on all available processors",
+      defaultValue = "0")
+  int warmup;
+
+  Path tempDir;
+
+  PrintWriter printWriter;
+
   public static void main(String[] args) {
     CommandLine cmd = new CommandLine(StorageSharedBenchmarkingCli.class);
     System.exit(cmd.execute(args));
   }
 
   @Override
   public void run() {
+    tempDir =
+        tempDirLocation != null
+            ? Paths.get(tempDirLocation)
+            : Paths.get(System.getProperty("java.io.tmpdir"));
+    printWriter = new PrintWriter(System.out, true);
     switch (testType) {
       case "w1r3":
         runWorkload1();
@@ -108,35 +120,66 @@ private void runWorkload1() {
     StorageOptions retryStorageOptions =
         StorageOptions.newBuilder().setProjectId(project).setRetrySettings(retrySettings).build();
     Storage storageClient = retryStorageOptions.getService();
-    runW1R3(storageClient);
+    try {
+      runW1R3(storageClient);
+    } catch (Exception e) {
+      System.err.println("Failed to run workload 1: " + e.getMessage());
+    }
   }
 
   private void runWorkload4() {
     RetrySettings retrySettings = StorageOptions.getDefaultRetrySettings().toBuilder().build();
     StorageOptions retryStorageOptions =
         StorageOptions.grpc().setRetrySettings(retrySettings).setAttemptDirectPath(true).build();
     Storage storageClient = retryStorageOptions.getService();
-    runW1R3(storageClient);
+    try {
+      runW1R3(storageClient);
+    } catch (Exception e) {
+      System.err.println("Failed to run workload 4: " + e.getMessage());
+    }
   }
 
-  private void runW1R3(Storage storageClient) {
-    Path tempDir =
-        tempDirLocation != null
-            ? Paths.get(tempDirLocation)
-            : Paths.get(System.getProperty("java.io.tmpdir"));
+  private void runW1R3(Storage storageClient) throws ExecutionException, InterruptedException {
     ListeningExecutorService executorService =
         MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(workers));
-    List<ApiFuture<String>> workloadRuns = new ArrayList<>();
-    Range objectSizeRange = Range.of(objectSize);
     for (int i = 0; i < samples; i++) {
+      runWarmup(storageClient);
+      Range objectSizeRange = Range.of(objectSize);
+      int objectSize = getRandomInt(objectSizeRange.min, objectSizeRange.max);
+      convert(
+              executorService.submit(
+                  new W1R3(
+                      storageClient,
+                      workers,
+                      api,
+                      printWriter,
+                      objectSize,
+                      tempDir,
+                      bucket,
+                      false)))
+          .get();
+    }
+  }
+
+  private void runWarmup(Storage storageClient) throws ExecutionException, InterruptedException {
+    if (warmup <= 0) {
+      return;
+    }
+    int numberProcessors = Runtime.getRuntime().availableProcessors();
+    ListeningExecutorService executorService =
+        MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numberProcessors));
+    long startTime = System.currentTimeMillis();
+    long endTime = startTime + (warmup * 1000);
+    // Run Warmup
+    while (System.currentTimeMillis() < endTime) {
+      Range objectSizeRange = Range.of(objectSize);
       int objectSize = getRandomInt(objectSizeRange.min, objectSizeRange.max);
-      PrintWriter pw = new PrintWriter(System.out, true);
-      workloadRuns.add(
-          convert(
+      convert(
               executorService.submit(
-                  new W1R3(storageClient, workers, api, pw, objectSize, tempDir, bucket))));
+                  new W1R3(
+                      storageClient, workers, api, printWriter, objectSize, tempDir, bucket, true)))
+          .get();
     }
-    ApiExceptions.callAndTranslateApiException(ApiFutures.allAsList(workloadRuns));
   }
 
   public static int getRandomInt(int min, int max) {
diff --git a/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/W1R3.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/W1R3.java
@@ -37,6 +37,7 @@ final class W1R3 implements Callable<String> {
   private final int objectSize;
   private final Path tempDirectory;
   private final String bucketName;
+  private final boolean isWarmup;
 
   W1R3(
       Storage storage,
@@ -45,18 +46,20 @@ final class W1R3 implements Callable<String> {
       PrintWriter printWriter,
       int objectSize,
       Path tempDirectory,
-      String bucketName) {
+      String bucketName,
+      boolean isWarmup) {
     this.storage = storage;
     this.workers = workers;
     this.api = api;
     this.printWriter = printWriter;
     this.objectSize = objectSize;
     this.tempDirectory = tempDirectory;
     this.bucketName = bucketName;
+    this.isWarmup = isWarmup;
   }
 
   @Override
-  public String call() throws Exception {
+  public String call() {
     // Create the file to be uploaded and fill it with data
 
     try (TmpFile file = DataGenerator.base64Characters().tempFile(tempDirectory, objectSize)) {
@@ -67,31 +70,50 @@ public String call() throws Exception {
       Blob created = storage.createFrom(blob, file.getPath());
       Instant endTime = clock.instant();
       Duration elapsedTimeUpload = Duration.between(startTime, endTime);
-      printWriter.println(
-          generateCloudMonitoringResult(
-                  "WRITE",
-                  StorageSharedBenchmarkingUtils.calculateThroughput(
-                      created.getSize().doubleValue(), elapsedTimeUpload),
-                  created)
-              .formatAsCustomMetric());
+      printResult("WRITE", created, elapsedTimeUpload);
       for (int i = 0; i <= StorageSharedBenchmarkingUtils.DEFAULT_NUMBER_OF_READS; i++) {
         try (TmpFile dest = TmpFile.of(tempDirectory, "prefix", "bin")) {
           startTime = clock.instant();
           storage.downloadTo(created.getBlobId(), dest.getPath());
           endTime = clock.instant();
           Duration elapsedTimeDownload = Duration.between(startTime, endTime);
-          printWriter.println(
-              generateCloudMonitoringResult(
-                      "READ[" + i + "]",
-                      StorageSharedBenchmarkingUtils.calculateThroughput(
-                          created.getSize().doubleValue(), elapsedTimeDownload),
-                      created)
-                  .formatAsCustomMetric());
+          printResult("READ[" + i + "]", created, elapsedTimeDownload);
         }
       }
       StorageSharedBenchmarkingUtils.cleanupObject(storage, created);
+      return "OK";
+    } catch (Exception e) {
+      CloudMonitoringResult result =
+          CloudMonitoringResult.newBuilder()
+              .setLibrary("java")
+              .setApi(api)
+              .setOp("W1R3")
+              .setWorkers(workers)
+              .setObjectSize(-1)
+              .setChunksize(-1)
+              .setCrc32cEnabled(false)
+              .setMd5Enabled(false)
+              .setCpuTimeUs(-1)
+              .setBucketName("")
+              .setStatus("FAIL")
+              .setTransferSize("")
+              .setThroughput(-1)
+              .build();
+      printWriter.println(result);
+      return "FAIL";
+    }
+  }
+
+  private void printResult(String op, Blob created, Duration duration) {
+    if (!isWarmup) {
+      printWriter.println(
+          generateCloudMonitoringResult(
+                  op,
+                  StorageSharedBenchmarkingUtils.calculateThroughput(
+                      created.getSize().doubleValue(), duration),
+                  created)
+              .formatAsCustomMetric());
     }
-    return "OK";
   }
 
   private CloudMonitoringResult generateCloudMonitoringResult(
