merge

.github/workflows/client-api-benchmark-evaluations.yml

@@ -1,4 +1,4 @@
-name: Run API + client benchmarks
+name: "[valor-service] benchmarks"
 
 on:
   push:

.github/workflows/client-api-tests-and-coverage.yml

@@ -1,4 +1,4 @@
-name: Run API + client code coverage report
+name: "[valor-service] code coverage report"
 
 on:
   push:
@@ -24,7 +24,7 @@ jobs:
       - name: set up postgres
         run: |
           docker build ./database -t pgvalor
-          docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -e POSTGRES_DB=valor -d pgvalor
+          docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -e POSTGRES_DB=valor -e POSTGRESQL_REPLICATION_USE_PASSFILE=false -d pgvalor
           sleep 3
           docker build ./migrations -t migrations
           docker run -e POSTGRES_PASSWORD=password -e POSTGRES_HOST=localhost -e POSTGRES_DB=valor -e POSTGRES_USERNAME=postgres -e POSTGRES_PORT=5432 --network "host" migrations

.github/workflows/lite-benchmark-evaluations.yml

@@ -1,4 +1,4 @@
-name: Run valor-lite benchmarks
+name: "[valor-lite] benchmarks"
 
 on:
   push:
@@ -35,4 +35,3 @@ jobs:
           export BENCHMARK_RESULTS=$(python -c "import os;import json;print(json.dumps(json.load(open('objdet_results.json', 'r')), indent=4));")
           echo "$BENCHMARK_RESULTS"
         working-directory: ./lite/benchmarks/
-      - run: make stop-env

.github/workflows/lite-synthetic-benchmarks.yml

@@ -0,0 +1,24 @@
+name: "[valor-lite] synthetic benchmarks"
+
+on:
+  push:
+    branches: "**"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  run-benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: install lite
+        run: pip install -e .
+        working-directory: ./lite
+      - name: benchmark semantic segmentation
+        run: python benchmark_semantic_segmentation.py
+        working-directory: ./lite/benchmarks/synthetic/

.github/workflows/lite-tests-and-coverage.yml

@@ -1,4 +1,4 @@
-name: Run valor-lite code coverage report
+name: "[valor-lite] code coverage report"
 
 on:
   push:

Makefile

@@ -13,7 +13,7 @@ unit-tests:
 
 start-postgres-docker:
 	docker build -t pgvalor ./database
-	docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -e POSTGRES_DB=valor -d pgvalor
+	docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -e POSTGRESQL_REPLICATION_USE_PASSFILE=false -e POSTGRES_DB=valor -d pgvalor
 
 run-migrations:
 ifeq ($(shell uname -s),Darwin)

api/.env.testing

@@ -3,3 +3,4 @@ POSTGRES_PASSWORD=password
 POSTGRES_HOST=db
 POSTGRES_DB=valor
 POSTGRES_PORT=5432
+POSTGRESQL_REPLICATION_USE_PASSFILE=false

docker-compose.yml

@@ -6,6 +6,7 @@ services:
     environment:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRESQL_REPLICATION_USE_PASSFILE: false
   service:
     image: ghcr.io/striveworks/valor/valor-service
     build: ./api

lite/benchmarks/synthetic/benchmark_semantic_segmentation.py

@@ -0,0 +1,94 @@
+from valor_lite.profiling import Benchmark, BenchmarkError
+from valor_lite.semantic_segmentation.benchmark import (
+    benchmark_add_data,
+    benchmark_evaluate,
+    benchmark_finalize,
+)
+
+
+def benchmark(
+    bitmask_shape: tuple[int, int],
+    number_of_unique_labels: int,
+    number_of_images: int,
+    *_,
+    memory_limit: float = 4.0,
+    time_limit: float = 10.0,
+    repeat: int = 1,
+    verbose: bool = False,
+):
+    """
+    Runs a single benchmark.
+
+    Parameters
+    ----------
+    bitmask_shape : tuple[int, int]
+        The size (h, w) of the bitmask array.
+    number_of_unique_labels : int
+        The number of unique labels used in the synthetic example.
+    number_of_images : int
+        The number of distinct datums that are created.
+    memory_limit : float
+        The maximum amount of system memory allowed in gigabytes (GB).
+    time_limit : float
+        The maximum amount of time permitted before killing the benchmark.
+    repeat : int
+        The number of times to run a benchmark to produce an average runtime.
+    verbose : bool, default=False
+        Toggles terminal output of benchmark results.
+    """
+
+    b = Benchmark(
+        time_limit=time_limit,
+        memory_limit=int(memory_limit * (1024**3)),
+        repeat=repeat,
+        verbose=verbose,
+    )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_add_data,
+        n_labels=[number_of_unique_labels],
+        shape=[bitmask_shape],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_finalize,
+        n_datums=[number_of_images],
+        n_labels=[number_of_unique_labels],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+    _, failed, details = b.run(
+        benchmark=benchmark_evaluate,
+        n_datums=[number_of_images],
+        n_labels=[number_of_unique_labels],
+    )
+    if failed:
+        raise BenchmarkError(
+            benchmark=details["benchmark"],
+            error_type=failed[0]["error"],
+            error_message=failed[0]["msg"],
+        )
+
+
+if __name__ == "__main__":
+
+    benchmark(
+        bitmask_shape=(4000, 4000),
+        number_of_images=1000,
+        number_of_unique_labels=10,
+        memory_limit=4.0,
+        time_limit=10.0,
+        repeat=1,
+        verbose=True,
+    )