Skip to content

Commit 21cb495

Browse files
authored
Fix pt2 dashboard passrate calculation regression (#6170)
The regression comes from #6006 where I changed `accuracy_results LEFT JOIN performance_results` to `performance_results LEFT JOIN accuracy_results` to accommodate Apple MPS eager benchmark. The swapped join wrongly returned `model_fail_to_load` and `eager_fail_to_run` models that shouldn't be included in the pass rate calculation because they are not something wrong with pt2. ### Testing https://torchci-git-fork-huydhn-fix-pt2-dashboard-p-e92d66-fbopensource.vercel.app/benchmark/compilers?dashboard=torchinductor&startTime=Fri%2C%2019%20Jul%202024%2000%3A29%3A48%20GMT&stopTime=Wed%2C%2015%20Jan%202025%2001%3A29%3A48%20GMT&granularity=week&mode=inference&dtype=bfloat16&deviceName=cuda%20(a100)&lBranch=main&lCommit=1dab79470dbecef79ba4c7d4308d8a181091e58e&rBranch=main&rCommit=b732b52f1e4378f8486ceb5e7026be3321c2651c * Before https://hud.pytorch.org/benchmark/torchbench/inductor_with_cudagraphs?dashboard=torchinductor&startTime=Thu%2C%2018%20Jul%202024%2023%3A36%3A11%20GMT&stopTime=Wed%2C%2015%20Jan%202025%2000%3A36%3A11%20GMT&granularity=week&mode=inference&dtype=bfloat16&deviceName=cuda%20(a100)&lBranch=main&lCommit=b732b52f1e4378f8486ceb5e7026be3321c2651c&rBranch=main&rCommit=b732b52f1e4378f8486ceb5e7026be3321c2651c * After https://torchci-git-fork-huydhn-fix-pt2-dashboard-p-e92d66-fbopensource.vercel.app/benchmark/torchbench/inductor_with_cudagraphs?dashboard=torchinductor&startTime=Thu%2C%2018%20Jul%202024%2023%3A36%3A11%20GMT&stopTime=Wed%2C%2015%20Jan%202025%2000%3A36%3A11%20GMT&granularity=week&mode=inference&dtype=bfloat16&deviceName=cuda%20(a100)&lBranch=main&lCommit=b732b52f1e4378f8486ceb5e7026be3321c2651c&rBranch=main&rCommit=b732b52f1e4378f8486ceb5e7026be3321c2651chttps%3A%2F%2Ftorchci-git-fork-huydhn-fix-pt2-dashboard-p-e92d66-fbopensource.vercel.app%2F
1 parent a74b323 commit 21cb495

File tree

1 file changed

+57
-3
lines changed
  • torchci/clickhouse_queries/compilers_benchmark_performance

1 file changed

+57
-3
lines changed

torchci/clickhouse_queries/compilers_benchmark_performance/query.sql

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,10 +83,8 @@ accuracy_results AS (
8383
workflow_id = { workflowId: Int64 }
8484
OR { workflowId: Int64 } = 0
8585
)
86-
AND accuracy != 'model_fail_to_load'
87-
AND accuracy != 'eager_fail_to_run'
8886
),
89-
results AS (
87+
performance_join_accuracy_results AS (
9088
SELECT
9189
performance_results.workflow_id AS workflow_id,
9290
performance_results.job_id AS job_id,
@@ -144,6 +142,62 @@ results AS (
144142
LEFT JOIN accuracy_results ON performance_results.name = accuracy_results.name
145143
AND performance_results.replaced_filename = accuracy_results.replaced_filename
146144
AND performance_results.workflow_id = accuracy_results.workflow_id
145+
WHERE
146+
accuracy != 'model_fail_to_load'
147+
AND accuracy != 'eager_fail_to_run'
148+
),
149+
-- This is to accommodate cases where only accuracy results are available, i.e. export
150+
accuracy_join_performance_results AS (
151+
SELECT
152+
accuracy_results.workflow_id AS workflow_id,
153+
accuracy_results.job_id AS job_id,
154+
CASE
155+
WHEN accuracy_results.replaced_filename LIKE '%_torchbench' THEN 'torchbench'
156+
WHEN accuracy_results.replaced_filename LIKE '%_timm_models' THEN 'timm_models'
157+
WHEN accuracy_results.replaced_filename LIKE '%_huggingface' THEN 'huggingface'
158+
ELSE ''
159+
END AS suite,
160+
CASE
161+
WHEN accuracy_results.replaced_filename LIKE '%_torchbench' THEN REPLACE(
162+
accuracy_results.replaced_filename,
163+
'_torchbench',
164+
''
165+
)
166+
WHEN accuracy_results.replaced_filename LIKE '%_timm_models' THEN REPLACE(
167+
accuracy_results.replaced_filename,
168+
'_timm_models',
169+
''
170+
)
171+
WHEN accuracy_results.replaced_filename LIKE '%_huggingface' THEN REPLACE(
172+
accuracy_results.replaced_filename,
173+
'_huggingface',
174+
''
175+
)
176+
ELSE ''
177+
END AS compiler,
178+
accuracy_results.name,
179+
0.0 AS speedup,
180+
accuracy,
181+
0.0 AS compilation_latency,
182+
0.0 AS compression_ratio,
183+
0.0 AS abs_latency,
184+
0.0 AS dynamo_peak_mem,
185+
0.0 AS eager_peak_mem,
186+
accuracy_results.timestamp AS timestamp
187+
FROM
188+
accuracy_results
189+
LEFT JOIN performance_results ON performance_results.name = accuracy_results.name
190+
AND performance_results.replaced_filename = accuracy_results.replaced_filename
191+
AND performance_results.workflow_id = accuracy_results.workflow_id
192+
WHERE
193+
performance_results.name = ''
194+
AND accuracy != 'model_fail_to_load'
195+
AND accuracy != 'eager_fail_to_run'
196+
),
197+
results AS (
198+
SELECT * FROM performance_join_accuracy_results
199+
UNION ALL
200+
SELECT * FROM accuracy_join_performance_results
147201
)
148202
SELECT
149203
DISTINCT results.workflow_id,

0 commit comments

Comments
 (0)