Added support for --unix-socket connection on CLI benchmark runner

.github/workflows/validate-spec-fields.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   pytest:
     name: Validate SPEC fields
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.259"
+version = "0.1.260"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__common__/runner.py

@@ -106,11 +106,16 @@ def extract_testsuites(args):
     testsuite_spec_files = get_benchmark_specs(
         testsuites_folder, args.test, args.tests_regexp
     )
+    testsuite_spec_files.sort()
     logging.info(
         "There are a total of {} test-suites in folder {}".format(
             len(testsuite_spec_files), testsuites_folder
         )
     )
+    if len(testsuite_spec_files) < 11:
+        for test in testsuite_spec_files:
+            logging.info(f"Test {test}")
+
     return testsuite_spec_files
 
 

redis_benchmarks_specification/__runner__/args.py

@@ -214,4 +214,9 @@ def create_client_runner_args(project_name):
         action="store_true",
         help="Run client in cluster mode.",
     )
+    parser.add_argument(
+        "--unix-socket",
+        default="",
+        help="UNIX Domain socket name",
+    )
     return parser

redis_benchmarks_specification/__runner__/runner.py

@@ -220,17 +220,26 @@ def prepare_memtier_benchmark_parameters(
     resp_version=None,
     override_memtier_test_time=0,
     override_test_runs=1,
+    unix_socket="",
 ):
     arbitrary_command = False
     benchmark_command = [
         full_benchmark_path,
-        "--port",
-        f"{port}",
-        "--server",
-        f"{server}",
         "--json-out-file",
         local_benchmark_output_filename,
     ]
+    if unix_socket != "":
+        benchmark_command.extend(["--unix-socket", unix_socket])
+        logging.info(f"Using UNIX SOCKET to connect {unix_socket}")
+    else:
+        benchmark_command.extend(
+            [
+                "--port",
+                f"{port}",
+                "--server",
+                f"{server}",
+            ]
+        )
     if password is not None:
         benchmark_command.extend(["--authenticate", password])
     if tls_enabled:
@@ -430,6 +439,7 @@ def delete_temporary_files(
 
                     port = args.db_server_port
                     host = args.db_server_host
+                    unix_socket = args.unix_socket
                     password = args.db_server_password
                     oss_cluster_api_enabled = args.cluster_mode
                     ssl_cert_reqs = "required"
@@ -606,7 +616,7 @@ def delete_temporary_files(
                                 )
                                 continue
                         if "preload_tool" in benchmark_config["dbconfig"]:
-                            if args.skip_tests_with_preload_via_tool is False:
+                            if args.skip_tests_with_preload_via_tool is True:
                                 logging.warning(
                                     "Skipping test {} giving it implies dataset preload via tool".format(
                                         test_name
@@ -648,6 +658,7 @@ def delete_temporary_files(
                                 args.benchmark_local_install,
                                 password,
                                 oss_cluster_api_enabled,
+                                unix_socket,
                             )
                             if res is False:
                                 logging.warning(
@@ -748,6 +759,7 @@ def delete_temporary_files(
                             resp_version,
                             override_memtier_test_time,
                             override_test_runs,
+                            unix_socket,
                         )
 
                     if (
@@ -1151,6 +1163,7 @@ def data_prepopulation_step(
     benchmark_local_install=False,
     password=None,
     oss_cluster_api_enabled=False,
+    unix_socket="",
 ):
     result = True
     # setup the benchmark
@@ -1193,6 +1206,8 @@ def data_prepopulation_step(
             tls_cacert,
             resp_version,
             override_memtier_test_time_preload,
+            1,
+            unix_socket,
         )
         if arbitrary_command is True and oss_cluster_api_enabled:
             logging.warning(

redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-string-get-10B-pipeline-100-nokeyprefix.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-10Mkeys-string-get-10B-pipeline-100-nokeyprefix
+description: Runs memtier_benchmark, for a keyspace length of 10M keys with a data size of 10 Bytes for each key.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 10000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '--key-maximum  10000000 -n allkeys --key-prefix "" --data-size 10 --ratio 1:0 --key-pattern P:P -c 50 -t 2 --hide-histogram --key-minimum 1'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- get
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--key-maximum  10000000 --key-minimum 1 --pipeline 100 --key-prefix "" --distinct-client-seed --data-size 10 --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 1

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100-nokeyprefix.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100-nokeyprefix
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs in which the value has a data size of 10 Bytes.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--pipeline" "100" "--data-size" "10" --distinct-client-seed --key-prefix "" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 120 -c 10 -t 10 --hide-histogram'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 17

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-100-nokeyprefix.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-get-10B-pipeline-100-nokeyprefix
+description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 10 Bytes for each key.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '--key-maximum  1000000 -n allkeys --key-prefix "" --data-size 10 --ratio 1:0 --key-pattern P:P -c 50 -t 2 --hide-histogram --key-minimum 1'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- get
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--key-maximum  1000000 --key-minimum 1 --pipeline 100 --key-prefix "" --distinct-client-seed --data-size 10 --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 1