fix: replace NaN values in deduped TA table (#1211)

Author: joseph-sentry

graphql_api/tests/test_test_analytics.py

@@ -21,6 +21,7 @@
     get_results,
 )
 from services.redis_configuration import get_redis_connection
+from utils.test_results import dedup_table
 
 from .helper import GraphQLTestHelper
 
@@ -218,7 +219,8 @@ def test_get_test_results(
     ):
         results = get_results(repository.repoid, repository.branch, 30)
         assert results is not None
-        assert results.equals(test_results_table)
+
+        assert results.equals(dedup_table(test_results_table))
 
     def test_get_test_results_no_storage(
         self, transactional_db, repository, mock_storage
@@ -231,7 +233,7 @@ def test_get_test_results_no_redis(
         m = mocker.patch("services.task.TaskService.cache_test_results_redis")
         results = get_results(repository.repoid, repository.branch, 30)
         assert results is not None
-        assert results.equals(test_results_table)
+        assert results.equals(dedup_table(test_results_table))
 
         m.assert_called_once_with(repository.repoid, repository.branch)
 

graphql_api/types/test_analytics/test_analytics.py

@@ -215,37 +215,6 @@ def generate_test_results(
             },
         )
 
-    failure_rate_expr = (
-        pl.col("failure_rate")
-        * (pl.col("total_fail_count") + pl.col("total_pass_count"))
-    ).sum() / (pl.col("total_fail_count") + pl.col("total_pass_count")).sum()
-
-    flake_rate_expr = (
-        pl.col("flake_rate") * (pl.col("total_fail_count") + pl.col("total_pass_count"))
-    ).sum() / (pl.col("total_fail_count") + pl.col("total_pass_count")).sum()
-
-    avg_duration_expr = (
-        pl.col("avg_duration")
-        * (pl.col("total_pass_count") + pl.col("total_fail_count"))
-    ).sum() / (pl.col("total_pass_count") + pl.col("total_fail_count")).sum()
-
-    # dedup
-    table = table.group_by("name").agg(
-        pl.col("test_id").first().alias("test_id"),
-        pl.col("testsuite").alias("testsuite"),
-        pl.col("flags").explode().unique().alias("flags"),
-        failure_rate_expr.alias("failure_rate"),
-        flake_rate_expr.alias("flake_rate"),
-        pl.col("updated_at").max().alias("updated_at"),
-        avg_duration_expr.alias("avg_duration"),
-        pl.col("total_fail_count").sum().alias("total_fail_count"),
-        pl.col("total_flaky_fail_count").sum().alias("total_flaky_fail_count"),
-        pl.col("total_pass_count").sum().alias("total_pass_count"),
-        pl.col("total_skip_count").sum().alias("total_skip_count"),
-        pl.col("commits_where_fail").sum().alias("commits_where_fail"),
-        pl.col("last_duration").max().alias("last_duration"),
-    )
-
     if term:
         table = table.filter(pl.col("name").str.contains(term))
 

utils/test_results.py

@@ -32,6 +32,45 @@ def storage_key(
     return key
 
 
+def dedup_table(table: pl.DataFrame) -> pl.DataFrame:
+    failure_rate_expr = (
+        pl.col("failure_rate")
+        * (pl.col("total_fail_count") + pl.col("total_pass_count"))
+    ).sum() / (pl.col("total_fail_count") + pl.col("total_pass_count")).sum()
+
+    flake_rate_expr = (
+        pl.col("flake_rate") * (pl.col("total_fail_count") + pl.col("total_pass_count"))
+    ).sum() / (pl.col("total_fail_count") + pl.col("total_pass_count")).sum()
+
+    avg_duration_expr = (
+        pl.col("avg_duration")
+        * (pl.col("total_pass_count") + pl.col("total_fail_count"))
+    ).sum() / (pl.col("total_pass_count") + pl.col("total_fail_count")).sum()
+
+    # dedup
+    table = (
+        table.group_by("name")
+        .agg(
+            pl.col("test_id").first().alias("test_id"),
+            pl.col("testsuite").alias("testsuite"),
+            pl.col("flags").explode().unique().alias("flags"),
+            failure_rate_expr.fill_nan(0).alias("failure_rate"),
+            flake_rate_expr.fill_nan(0).alias("flake_rate"),
+            pl.col("updated_at").max().alias("updated_at"),
+            avg_duration_expr.fill_nan(0).alias("avg_duration"),
+            pl.col("total_fail_count").sum().alias("total_fail_count"),
+            pl.col("total_flaky_fail_count").sum().alias("total_flaky_fail_count"),
+            pl.col("total_pass_count").sum().alias("total_pass_count"),
+            pl.col("total_skip_count").sum().alias("total_skip_count"),
+            pl.col("commits_where_fail").sum().alias("commits_where_fail"),
+            pl.col("last_duration").max().alias("last_duration"),
+        )
+        .sort("name")
+    )
+
+    return table
+
+
 def get_results(
     repoid: int,
     branch: str,
@@ -73,4 +112,6 @@ def get_results(
     if table.height == 0:
         return None
 
+    table = dedup_table(table)
+
     return table