Skip to content

Commit 18282b1

Browse files
qti-jkilpatrickjeffkilpatrick
qti-jkilpatrick
and
jeffkilpatrick
authored
[QNN EP] Revert workarounds for problems in old QNN versions (#25171)
### Description * Re-enable tests and remove workarounds that were introduced as part of a QNN <= 2.31 upgrade but are no longer necessary. ### Motivation and Context QNN/QAIRT releases about once a month. As ONNX Runtime adopts these new versions, some number of tests are often found to be impacted. Consequently, tests are skipped and tolerances are loosened. This change reverts as many of those workarounds as possible that were made for QNN upgrades between 2.17 and 2.31, inclusive. The most recent few releases were intentionally not examined to minimize impact on users on old versions and to avoid lock-in to the bleeding edge. --------- Co-authored-by: Jeff Kilpatrick <[email protected]>
1 parent 4094f92 commit 18282b1

File tree

9 files changed

+23
-145
lines changed

9 files changed

+23
-145
lines changed

‎onnxruntime/test/onnx/TestCase.cc

Lines changed: 1 addition & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1393,59 +1393,7 @@ std::unique_ptr<std::set<BrokenTest>> GetBrokenTests(const std::string& provider
13931393
}
13941394

13951395
if (provider_name == "qnn") {
1396-
broken_tests->insert({"gemm_default_no_bias", "result differs"});
13971396
broken_tests->insert({"resize_downsample_scales_linear", "result differs"});
1398-
broken_tests->insert({"resize_downsample_scales_linear_antialias", "result differs"});
1399-
broken_tests->insert({"resize_downsample_sizes_linear_antialias", "result differs"});
1400-
broken_tests->insert({"sce_NCd1_mean_weight_negative_ii", "result differs"});
1401-
broken_tests->insert({"sce_NCd1_mean_weight_negative_ii_expanded", "result differs"});
1402-
broken_tests->insert({"sce_NCd1_mean_weight_negative_ii_log_prob", "result differs"});
1403-
broken_tests->insert({"sce_NCd1_mean_weight_negative_ii_log_prob_expanded", "result differs"});
1404-
broken_tests->insert({"sce_mean", "result differs"});
1405-
broken_tests->insert({"sce_mean_3d", "result differs"});
1406-
broken_tests->insert({"sce_mean_3d_expanded", "result differs"});
1407-
broken_tests->insert({"sce_mean_3d_log_prob", "result differs"});
1408-
broken_tests->insert({"sce_mean_3d_log_prob_expanded", "result differs"});
1409-
broken_tests->insert({"sce_mean_expanded", "result differs"});
1410-
broken_tests->insert({"sce_mean_log_prob", "result differs"});
1411-
broken_tests->insert({"sce_mean_log_prob_expanded", "result differs"});
1412-
broken_tests->insert({"sce_mean_no_weight_ii", "result differs"});
1413-
broken_tests->insert({"sce_mean_no_weight_ii_3d", "result differs"});
1414-
broken_tests->insert({"sce_mean_no_weight_ii_3d_expanded", "result differs"});
1415-
broken_tests->insert({"sce_mean_no_weight_ii_3d_log_prob", "result differs"});
1416-
broken_tests->insert({"sce_mean_no_weight_ii_3d_log_prob_expanded", "result differs"});
1417-
broken_tests->insert({"sce_mean_no_weight_ii_4d", "result differs"});
1418-
broken_tests->insert({"sce_mean_no_weight_ii_4d_expanded", "result differs"});
1419-
broken_tests->insert({"sce_mean_no_weight_ii_4d_log_prob", "result differs"});
1420-
broken_tests->insert({"sce_mean_no_weight_ii_4d_log_prob_expanded", "result differs"});
1421-
broken_tests->insert({"sce_mean_no_weight_ii_expanded", "result differs"});
1422-
broken_tests->insert({"sce_mean_no_weight_ii_log_prob", "result differs"});
1423-
broken_tests->insert({"sce_mean_no_weight_ii_log_prob_expanded", "result differs"});
1424-
broken_tests->insert({"sce_mean_weight", "result differs"});
1425-
broken_tests->insert({"sce_mean_weight_expanded", "result differs"});
1426-
broken_tests->insert({"sce_mean_weight_ii", "result differs"});
1427-
broken_tests->insert({"sce_mean_weight_ii_3d", "result differs"});
1428-
broken_tests->insert({"sce_mean_weight_ii_3d_expanded", "result differs"});
1429-
broken_tests->insert({"sce_mean_weight_ii_3d_log_prob", "result differs"});
1430-
broken_tests->insert({"sce_mean_weight_ii_3d_log_prob_expanded", "result differs"});
1431-
broken_tests->insert({"sce_mean_weight_ii_4d", "result differs"});
1432-
broken_tests->insert({"sce_mean_weight_ii_4d_expanded", "result differs"});
1433-
broken_tests->insert({"sce_mean_weight_ii_4d_log_prob", "result differs"});
1434-
broken_tests->insert({"sce_mean_weight_ii_4d_log_prob_expanded", "result differs"});
1435-
broken_tests->insert({"sce_mean_weight_ii_expanded", "result differs"});
1436-
broken_tests->insert({"sce_mean_weight_ii_log_prob", "result differs"});
1437-
broken_tests->insert({"sce_mean_weight_ii_log_prob_expanded", "result differs"});
1438-
broken_tests->insert({"sce_mean_weight_log_prob", "result differs"});
1439-
broken_tests->insert({"sce_mean_weight_log_prob_expanded", "result differs"});
1440-
broken_tests->insert({"sce_none", "result differs"});
1441-
broken_tests->insert({"sce_none_expanded", "result differs"});
1442-
broken_tests->insert({"sce_none_log_prob", "result differs"});
1443-
broken_tests->insert({"sce_none_log_prob_expanded", "result differs"});
1444-
broken_tests->insert({"sce_sum", "result differs"});
1445-
broken_tests->insert({"sce_sum_expanded", "result differs"});
1446-
broken_tests->insert({"sce_sum_log_prob", "result differs"});
1447-
broken_tests->insert({"sce_sum_log_prob_expanded", "result differs"});
1448-
broken_tests->insert({"gridsample_reflection_padding", "result differs"});
14491397
broken_tests->insert({"gridsample_volumetric_nearest_align_corners_0", "unknown version"});
14501398
broken_tests->insert({"gridsample_volumetric_nearest_align_corners_1", "unknown version"});
14511399
broken_tests->insert({"rotary_embedding", "unknown version"});
@@ -1454,9 +1402,7 @@ std::unique_ptr<std::set<BrokenTest>> GetBrokenTests(const std::string& provider
14541402
broken_tests->insert({"rotary_embedding_no_position_ids_expanded", "unknown version"});
14551403
broken_tests->insert({"rotary_embedding_no_position_ids_interleaved", "unknown version"});
14561404
broken_tests->insert({"rotary_embedding_no_position_ids_interleaved_expanded", "unknown version"});
1457-
broken_tests->insert({"spacetodepth", "result differs"});
1458-
broken_tests->insert({"reduce_sum_square_empty_set_expanded", "unknown version"});
1459-
// Fails with QNN SDK 2.17.0:
1405+
// Fails since QNN SDK 2.17.0:
14601406
// expected 7.70947 (40f6b3f3), got 7.84096 (40fae920), diff: 0.131491, tol=0.00870947 idx=419. 100 of 1715 differ
14611407
broken_tests->insert({"facedetection_op8_qdq", "result differs"});
14621408
// Fails with QNN SDK 2.34.0:
@@ -1466,11 +1412,6 @@ std::unique_ptr<std::set<BrokenTest>> GetBrokenTests(const std::string& provider
14661412
broken_tests->insert({"mobilenetv2-1.0", "result differs with 2.34"});
14671413
broken_tests->insert({"facedetection_op8", "segfault with CPU backend, will be fixed by QNN 2.36"});
14681414

1469-
#if defined(_WIN32) && defined(_M_AMD64)
1470-
// Fails with QNN SDK 2.17.0 on Windows x64:
1471-
// expected 13.5 (41580000), got 0 (0), diff: 13.5, tol=0.0145 idx=3. 3 of 4 differ
1472-
broken_tests->insert({"averagepool_2d_ceil", "result differs"});
1473-
#endif
14741415
// These next 3 Resize tests fail on CPU backend with QNN SDK 2.22.0 due to inaccuracy.
14751416
// output=Y:expected 1 (3f800000), got 3 (40400000), diff: 2, tol=0.002 idx=24. 8 of 56 differ
14761417
broken_tests->insert({"resize_upsample_sizes_nearest", "result differs"});
@@ -1482,12 +1423,6 @@ std::unique_ptr<std::set<BrokenTest>> GetBrokenTests(const std::string& provider
14821423
broken_tests->insert({"convtranspose_group_2_image_3", "Segmentation fault (core dumped). CPU test passed."});
14831424
// Fails with QNN 2.31 on Windows x64 for CPU
14841425
broken_tests->insert({"gelu_tanh_2", "y:expected -0.0131778 (bc57e7d5), got -0.0136333 (bc5f5e38), diff: 0.000455472, tol=2.31778e-05."});
1485-
broken_tests->insert({"convtranspose_pad", "Access violation 0xc000005 from call graphAddNode."});
1486-
broken_tests->insert({"convtranspose_pads", "Access violation 0xc000005 from call graphAddNode."});
1487-
broken_tests->insert({"convtranspose_output_shape", "Access violation 0xc000005 from call graphAddNode."});
1488-
broken_tests->insert({"convtranspose_kernel_shape", "Access violation 0xc000005 from call graphAddNode."});
1489-
broken_tests->insert({"convtranspose_1d", "Access violation 0xc000005 from call graphAddNode."});
1490-
broken_tests->insert({"convtranspose", "Access violation 0xc000005 from call graphAddNode."});
14911426
broken_tests->insert({"averagepool_2d_ceil", "result differs. expected 13.5 (41580000), got 0 (0)"});
14921427
// Fails with QNN 2.32
14931428
broken_tests->insert({"resize_upsample_scales_linear", "expected 1 (3f800000), got 0.25 (3e800000)"});

‎onnxruntime/test/providers/qnn/average_pool_test.cc

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,7 @@ TEST_F(QnnHTPBackendTests, AveragePool_CountIncludePad_HTP_u8) {
142142
{utils::MakeAttribute("kernel_shape", std::vector<int64_t>{1, 1}),
143143
utils::MakeAttribute("count_include_pad", static_cast<int64_t>(1))},
144144
ExpectedEPNodeAssignment::All,
145-
18,
146-
// Need tolerance of 0.414% of output range after QNN SDK 2.17
147-
QDQTolerance(0.00414f));
145+
18);
148146
}
149147

150148
// QDQ AveragePool that use auto_pad 'SAME_UPPER'.
@@ -157,9 +155,7 @@ TEST_F(QnnHTPBackendTests, AveragePool_AutopadSameUpper_HTP_u8) {
157155
{utils::MakeAttribute("kernel_shape", std::vector<int64_t>{1, 1}),
158156
utils::MakeAttribute("auto_pad", "SAME_UPPER")},
159157
ExpectedEPNodeAssignment::All,
160-
18,
161-
// Need to use tolerance of 0.414% of output range after QNN SDK 2.17
162-
QDQTolerance(0.00414f));
158+
18);
163159
}
164160

165161
// QDQ AveragePool that use auto_pad 'SAME_LOWER'.
@@ -172,9 +168,7 @@ TEST_F(QnnHTPBackendTests, AveragePool_AutopadSameLower_HTP_u8) {
172168
{utils::MakeAttribute("kernel_shape", std::vector<int64_t>{1, 1}),
173169
utils::MakeAttribute("auto_pad", "SAME_LOWER")},
174170
ExpectedEPNodeAssignment::All,
175-
18,
176-
// Need to use tolerance of 0.414% of output range after QNN SDK 2.17
177-
QDQTolerance(0.00414f));
171+
18);
178172
}
179173

180174
#endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)

‎onnxruntime/test/providers/qnn/conv_test.cc

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -708,9 +708,7 @@ TEST_F(QnnHTPBackendTests, DISABLED_Test_QDQConvWithDynamicWeightsFromMul) {
708708
RunQnnModelTest(BuildConvMulGraph,
709709
provider_options,
710710
13,
711-
ExpectedEPNodeAssignment::All,
712-
4e-4f); // Accuracy decreased slightly in QNN SDK 2.17.
713-
// Expected: 9.94500065, Actual: 9.94537735
711+
ExpectedEPNodeAssignment::All);
714712
}
715713

716714
// Check that QNN compiles DQ -> Conv -> Q as a single unit.
@@ -727,9 +725,7 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_dynamic_input) {
727725
"NOTSET",
728726
ExpectedEPNodeAssignment::All,
729727
false, // use_qdq_contrib_ops
730-
13, // opset
731-
// Need tolerance of 0.413% of output range after QNN SDK 2.17
732-
QDQTolerance(0.00413f));
728+
13); // opset
733729

734730
RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
735731
TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.0f, 10.0f), // Random dynamic input
@@ -742,9 +738,7 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_dynamic_input) {
742738
"NOTSET",
743739
ExpectedEPNodeAssignment::All,
744740
false, // use_qdq_contrib_ops
745-
13, // opset
746-
// Need tolerance of 0.413% of output range after QNN SDK 2.17
747-
QDQTolerance(0.00413f));
741+
13); // opset
748742
}
749743

750744
// Test per-channel QDQ Conv. in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
@@ -1911,9 +1905,7 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_initializer) {
19111905
"NOTSET",
19121906
ExpectedEPNodeAssignment::All,
19131907
false, // use_qdq_contrib_ops
1914-
13, // opset
1915-
// Need tolerance of 0.413% of output range after QNN SDK 2.17
1916-
QDQTolerance(0.00413f));
1908+
13); // opset
19171909

19181910
RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
19191911
TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.0f, 10.0f), // Random dynamic input
@@ -1926,9 +1918,7 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_initializer) {
19261918
"NOTSET",
19271919
ExpectedEPNodeAssignment::All,
19281920
false, // use_qdq_contrib_ops
1929-
13, // opset
1930-
// Need tolerance of 0.413% of output range after QNN SDK 2.17
1931-
QDQTolerance(0.00413f));
1921+
13); // opset
19321922
}
19331923

19341924
// Tests 1D Conv with bias as an initializer.
@@ -2136,12 +2126,6 @@ TEST_F(QnnHTPBackendTests, DISABLED_ConvU8U8S32_large_input1_padding_bias_initia
21362126
}
21372127

21382128
TEST_F(QnnHTPBackendTests, ConvU8U8S32_large_input2_bias_initializer) {
2139-
#ifdef __linux__
2140-
// On Linux QNN SDK 2.17: Need a tolerance of 0.785% of output range to pass.
2141-
QDQTolerance tolerance = QDQTolerance(0.00785f);
2142-
#else
2143-
QDQTolerance tolerance = QDQTolerance();
2144-
#endif
21452129
RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
21462130
TestInputDef<float>({1, 128, 8, 56}, false, 0.f, 10.f), // Dynamic input
21472131
TestInputDef<float>({32, 128, 1, 1}, true, -1.f, 1.f), // Random static weights
@@ -2153,8 +2137,7 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_large_input2_bias_initializer) {
21532137
"NOTSET",
21542138
ExpectedEPNodeAssignment::All,
21552139
false,
2156-
13,
2157-
tolerance);
2140+
13);
21582141
}
21592142

21602143
TEST_F(QnnHTPBackendTests, ConvU8U8S32_LargeInput_Dilations_Pads) {

‎onnxruntime/test/providers/qnn/gemm_op_test.cc

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -336,8 +336,7 @@ TEST_F(QnnHTPBackendTests, Gemm_Broadcast_Bias_DynamicInputs) {
336336
ExpectedEPNodeAssignment::All,
337337
13,
338338
false,
339-
// Require tolerance of 0.74% on Windows ARM64.
340-
QDQTolerance(0.0074f));
339+
QDQTolerance(0.00410f));
341340
}
342341

343342
TEST_F(QnnHTPBackendTests, Gemm_Broadcast_Bias_DynamicA_StaticB_DynamicC) {
@@ -356,8 +355,7 @@ TEST_F(QnnHTPBackendTests, Gemm_Broadcast_Bias_DynamicA_StaticB_DynamicC) {
356355
ExpectedEPNodeAssignment::All,
357356
13,
358357
false,
359-
// Require tolerance of 0.74% on Windows ARM64.
360-
QDQTolerance(0.0074f));
358+
QDQTolerance(0.00410f));
361359
}
362360

363361
TEST_F(QnnHTPBackendTests, Gemm_Broadcast_Bias_DynamicA_StaticB_StaticC) {
@@ -376,8 +374,7 @@ TEST_F(QnnHTPBackendTests, Gemm_Broadcast_Bias_DynamicA_StaticB_StaticC) {
376374
ExpectedEPNodeAssignment::All,
377375
13,
378376
false,
379-
// Require tolerance of 0.74% on Windows ARM64.
380-
QDQTolerance(0.0074f));
377+
QDQTolerance(0.00410f));
381378
}
382379

383380
// Test 16-bit QDQ Gemm with dynamic inputs A and Bias. The B input is an initializer.

‎onnxruntime/test/providers/qnn/layer_norm_test.cc

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,7 @@ static void RunLayerNormCpuTest(const TestInputDef<float>& input_def,
3232
expected_ep_assignment);
3333
}
3434

35-
#ifdef __linux__
36-
// This CPU test fails on Linux, QNN SDK 2.17
37-
// the value pair (-1.75661933, 0) at index #1 don't match, which is 1.75662 from -1.75662
38-
TEST_F(QnnCPUBackendTests, DISABLED_LayerNorm) {
39-
#else
4035
TEST_F(QnnCPUBackendTests, LayerNorm) {
41-
#endif
4236
RunLayerNormCpuTest(TestInputDef<float>({2, 3}, false, GetFloatDataInRange(0.0f, 10.0f, 6)),
4337
TestInputDef<float>({2, 3}, false, GetFloatDataInRange(0.0f, 10.0f, 6)),
4438
{utils::MakeAttribute("axis", static_cast<int64_t>(0))},
@@ -210,7 +204,7 @@ TEST_F(QnnHTPBackendTests, LayerNorm1D_LastAxis_StaticScale_AU16_WU8) {
210204

211205
// Test accuracy of 8-bit QDQ LayerNorm with a dynamic scale input.
212206
//
213-
// TODO(adrianlizarraga): Fails to finalize with QNN SDK 2.22. Still fails on QNN SDK 2.28.2.
207+
// TODO(adrianlizarraga): Fails to finalize with QNN SDK 2.22. Still fails on QNN SDK 2.35.0.
214208
// Verbose logs:
215209
// Starting stage: Graph Transformations and Optimizations
216210
// C:\...\QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::flat_to_vtcm

‎onnxruntime/test/providers/qnn/lrn_op_test.cc

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -149,20 +149,13 @@ TEST_F(QnnHTPBackendTests, LRNSize5) {
149149
}
150150

151151
TEST_F(QnnHTPBackendTests, LRN_size_larger_than_channel) {
152-
#ifdef __linux__
153-
// On Linux QNN SDK 2.17: Need a tolerance of 0.407% of output range to pass.
154-
QDQTolerance tolerance = QDQTolerance(0.00407f);
155-
#else
156-
QDQTolerance tolerance = QDQTolerance();
157-
#endif
158152
RunQDQLRNOpTest<uint8_t>(TestInputDef<float>({1, 128, 4, 5}, false, -10.0f, 10.0f),
159153
255, // Size
160154
ExpectedEPNodeAssignment::All,
161155
0.0001f, // alpha
162156
0.75f, // beta
163157
1.0f, // bias
164-
13, // opset
165-
tolerance);
158+
13);
166159
}
167160

168161
#endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)

‎onnxruntime/test/providers/qnn/matmul_test.cpp

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -194,13 +194,7 @@ TEST_F(QnnCPUBackendTests, MatMulOp) {
194194
RunMatMulOpTest(false, {2, 3, 3, 3}, {3, 2}, false, true);
195195
RunMatMulOpTest(false, {2, 3, 3, 3}, {2, 3, 3, 2}, false, true);
196196

197-
#if defined(__linux__)
198-
// TODO: This fails on Linux (HTP emulation). Works on Windows ARM64.
199-
// Expected: contains 24 values, where each value and its corresponding value in 16-byte object <18-00 00-00 00-00 00-00 00-29 4E-53 A8-55 00-00> are an almost-equal pair
200-
// Actual: 16-byte object <18-00 00-00 00-00 00-00 80-28 3E-53 A8-55 00-00>, where the value pair (0.0285999943, 0) at index #12 don't match, which is -0.0286 from 0.0286
201-
#else
202197
RunMatMulOpTest(false, {2, 1, 2, 3}, {3, 3, 2}, false, false);
203-
#endif
204198
RunMatMulOpTest(false, {3}, {3}, false, false);
205199
RunMatMulOpTest(false, {3}, {3}, false, true);
206200
RunMatMulOpTest(false, {3}, {3}, true, false);
@@ -285,7 +279,7 @@ TEST_F(QnnHTPBackendTests, MatMulOp_QDQ) {
285279
// UINT16, per-channel INT8 weight
286280
RunQDQPerChannelMatMulOpTest<uint16_t, int8_t, uint16_t>({2, 3}, {3, 2}, 1, QDQTolerance(),
287281
ExpectedEPNodeAssignment::All, 21, false, false);
288-
RunQDQPerChannelMatMulOpTest<uint16_t, int8_t, uint16_t>({2, 3, 3}, {3}, -1, QDQTolerance(0.005f));
282+
RunQDQPerChannelMatMulOpTest<uint16_t, int8_t, uint16_t>({2, 3, 3}, {3}, -1, QDQTolerance(0.0041f));
289283
}
290284

291285
// Tests MatMul with two uint16 (quantized) inputs that are both dynamic.

‎onnxruntime/test/providers/qnn/pool_op_test.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -182,10 +182,8 @@ TEST_F(QnnHTPBackendTests, MaxPool_Large_Input_HTP_u8) {
182182
utils::MakeAttribute("storage_order", static_cast<int64_t>(0)),
183183
utils::MakeAttribute("auto_pad", "NOTSET")},
184184
ExpectedEPNodeAssignment::All,
185-
18, // opset
186-
false, // use_contrib_qdq_ops
187-
// Need a tolerance of 0.417% of output range after QNN SDK 2.17
188-
QDQTolerance(0.00417f));
185+
18, // opset
186+
false); // use_contrib_qdq_ops
189187
}
190188

191189
TEST_F(QnnHTPBackendTests, MaxPool1D_ReshapeNodesPresent) {

0 commit comments

Comments
 (0)