Enabled passing baseline/comparison hash and github_repo to ensure proper data filtering on compare. Removed refs/heads/ usage from builder (#265)

* Enabled running forks source built benchmarks

* Fixed server_name Null check

* Enabled passing baseline/comparison hash and github_repo to ensure proper data filtering on compare. Removed refs/heads/ usage from builder

* skipping cli builder test on ci

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.217"
+version = "0.1.218"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__cli__/cli.py

@@ -351,16 +351,22 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
                 hash_regexp
             )
         )
+    enable_hash_filtering = False
+    hash_filters = []
+    if args.git_hash != "":
+        enable_hash_filtering = True
+        hash_filters = args.git_hash.split(",")
+        logging.info(
+            f"There is a total of {len(hash_filters)} commit hash fitlers: {hash_filters}"
+        )
     hash_regexp_string = re.compile(hash_regexp)
     filtered_hash_commits = []
     for cdict in commits:
         commit_hash = cdict["git_hash"]
-        if args.git_hash != "":
-            if args.git_hash != commit_hash:
+        if enable_hash_filtering:
+            if commit_hash not in hash_filters:
                 logging.info(
-                    "Skipping {} given it does not match commit hash {}".format(
-                        commit_hash, args.git_hash
-                    )
+                    f"Skipping {commit_hash} given it does not match any commit hash in {hash_filters}"
                 )
                 continue
         commit_summary = cdict["commit_summary"]

redis_benchmarks_specification/__common__/builder_schema.py

@@ -174,6 +174,10 @@ def get_branch_version_from_test_details(testDetails):
             git_branch = git_branch.decode()
         if git_branch.startswith("/refs/heads/"):
             git_branch = git_branch.replace("/refs/heads/", "")
+        if git_branch.startswith("refs/heads/"):
+            git_branch = git_branch.replace("refs/heads/", "")
+        if git_branch.startswith("/"):
+            git_branch = git_branch[1:]
     if git_version is not None:
         if type(git_version) == bytes:
             git_version = git_version.decode()

redis_benchmarks_specification/__compare__/args.py

@@ -92,11 +92,19 @@ def create_compare_arguments(parser):
     )
     parser.add_argument("--baseline-branch", type=str, default=None, required=False)
     parser.add_argument("--baseline-tag", type=str, default=None, required=False)
+    parser.add_argument("--baseline-hash", type=str, default=None, required=False)
     parser.add_argument(
         "--baseline-target-version", type=str, default=None, required=False
     )
     parser.add_argument("--comparison-branch", type=str, default=None, required=False)
+    parser.add_argument(
+        "--baseline-github-repo", type=str, default="redis", required=False
+    )
+    parser.add_argument(
+        "--comparison-github-repo", type=str, default="redis", required=False
+    )
     parser.add_argument("--comparison-tag", type=str, default=None, required=False)
+    parser.add_argument("--comparison-hash", type=str, default=None, required=False)
     parser.add_argument(
         "--comparison-target-version", type=str, default=None, required=False
     )

redis_benchmarks_specification/__compare__/compare.py

@@ -250,6 +250,10 @@ def compare_command_logic(args, project_name, project_version):
     running_platform = args.running_platform
     baseline_target_version = args.baseline_target_version
     comparison_target_version = args.comparison_target_version
+    baseline_github_repo = args.baseline_github_repo
+    comparison_github_repo = args.comparison_github_repo
+    baseline_hash = args.baseline_hash
+    comparison_hash = args.comparison_hash
 
     if running_platform is not None:
         logging.info(
@@ -310,6 +314,10 @@ def compare_command_logic(args, project_name, project_version):
         running_platform,
         baseline_target_version,
         comparison_target_version,
+        baseline_hash,
+        comparison_hash,
+        baseline_github_repo,
+        comparison_github_repo,
     )
     prepare_regression_comment(
         auto_approve,
@@ -535,6 +543,10 @@ def compute_regression_table(
     running_platform=None,
     baseline_target_version=None,
     comparison_target_version=None,
+    comparison_hash=None,
+    baseline_hash=None,
+    baseline_github_repo="redis",
+    comparison_github_repo="redis",
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -560,6 +572,8 @@ def compute_regression_table(
         comparison_tag,
         baseline_target_version,
         comparison_target_version,
+        comparison_hash,
+        baseline_hash,
     )
     logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}")
     logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}")
@@ -605,6 +619,8 @@ def compute_regression_table(
         total_stable,
         total_unstable,
         total_comparison_points,
+        regressions_list,
+        improvements_list,
     ) = from_rts_to_regression_table(
         baseline_deployment_name,
         comparison_deployment_name,
@@ -629,6 +645,8 @@ def compute_regression_table(
         tf_triggering_env,
         verbose,
         running_platform,
+        baseline_github_repo,
+        comparison_github_repo,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -661,6 +679,8 @@ def compute_regression_table(
         writer_regressions.dump(mystdout, False)
         table_output += mystdout.getvalue()
         table_output += "\n\n"
+        test_names_str = "|".join(regressions_list)
+        table_output += f"Regressions test regexp names: {test_names_str}\n\n"
         mystdout.close()
         sys.stdout = old_stdout
 
@@ -682,6 +702,8 @@ def compute_regression_table(
         writer_regressions.dump(mystdout, False)
         table_output += mystdout.getvalue()
         table_output += "\n\n"
+        test_names_str = "|".join(improvements_list)
+        table_output += f"Improvements test regexp names: {test_names_str}\n\n"
         mystdout.close()
         sys.stdout = old_stdout
 
@@ -724,6 +746,8 @@ def get_by_strings(
     comparison_tag,
     baseline_target_version=None,
     comparison_target_version=None,
+    baseline_hash=None,
+    comparison_hash=None,
 ):
     baseline_covered = False
     comparison_covered = False
@@ -760,6 +784,16 @@ def get_by_strings(
         by_str_baseline = "target+version"
         baseline_str = baseline_target_version
 
+    if baseline_hash is not None:
+        if comparison_covered:
+            logging.error(
+                "--baseline-branch, --baseline-tag, --baseline-hash, and --baseline-target-version are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        baseline_covered = True
+        by_str_baseline = "hash"
+        baseline_str = baseline_hash
+
     if comparison_tag is not None:
         # check if we had already covered comparison
         if comparison_covered:
@@ -781,16 +815,27 @@ def get_by_strings(
         by_str_comparison = "target+version"
         comparison_str = comparison_target_version
 
+    if comparison_hash is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, --comparison-hash, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "hash"
+        comparison_str = comparison_hash
+
     if baseline_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --baseline-branch, --baseline-tag, or --baseline-target-version ) "
+            + "( --baseline-branch, --baseline-tag, --baseline-hash, or --baseline-target-version ) "
         )
         exit(1)
     if comparison_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --comparison-branch, --comparison-tag, or --comparison-target-version ) "
+            + "( --comparison-branch, --comparison-tag, --comparison-hash, or --comparison-target-version ) "
         )
         exit(1)
     return baseline_str, by_str_baseline, comparison_str, by_str_comparison
@@ -820,6 +865,8 @@ def from_rts_to_regression_table(
     tf_triggering_env,
     verbose,
     running_platform=None,
+    baseline_github_repo="redis",
+    comparison_github_repo="redis",
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table_full = []
@@ -835,6 +882,8 @@ def from_rts_to_regression_table(
     total_comparison_points = 0
     noise_waterline = 3
     progress = tqdm(unit="benchmark time-series", total=len(test_names))
+    regressions_list = []
+    improvements_list = []
     for test_name in test_names:
         compare_version = "v0.1.208"
         github_link = "https://github.com/redis/redis-benchmarks-specification/blob"
@@ -848,18 +897,23 @@ def from_rts_to_regression_table(
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(baseline_deployment_name),
+            "github_repo={}".format(baseline_github_repo),
             "triggering_env={}".format(tf_triggering_env),
         ]
         if running_platform is not None:
             filters_baseline.append("running_platform={}".format(running_platform))
         filters_comparison = [
             "{}={}".format(by_str_comparison, comparison_str),
             "metric={}".format(metric_name),
-            "hash==",
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(comparison_deployment_name),
+            "github_repo={}".format(comparison_github_repo),
             "triggering_env={}".format(tf_triggering_env),
         ]
+        if "hash" not in by_str_baseline:
+            filters_baseline.append("hash==")
+        if "hash" not in by_str_comparison:
+            filters_comparison.append("hash==")
         if running_platform is not None:
             filters_comparison.append("running_platform={}".format(running_platform))
         baseline_timeseries = rts.ts().queryindex(filters_baseline)
@@ -1037,9 +1091,11 @@ def from_rts_to_regression_table(
                 test_link,
             )
             if detected_regression:
+                regressions_list.append(test_name)
                 table_regressions.append(line)
 
             if detected_improvement:
+                improvements_list.append(test_name)
                 table_improvements.append(line)
 
             if unstable:
@@ -1072,6 +1128,8 @@ def from_rts_to_regression_table(
         total_stable,
         total_unstable,
         total_comparison_points,
+        regressions_list,
+        improvements_list,
     )
 
 

utils/tests/test_builder.py

@@ -154,9 +154,7 @@ def test_get_branch_version_from_test_details():
 
 def test_cli_build():
     try:
-        # if should_run_builder():
-        if True:
-
+        if should_run_builder():
             db_port = int(os.getenv("DATASINK_PORT", "6379"))
             conn = redis.StrictRedis(port=db_port)
             conn.ping()