Skip to content

Commit 9ed0238

Browse files
MartinPavellaMartinPavella
authored
NXP backend: Use a quantized dataset for testing operators where a single-bit error is expected. (#20689)
### Summary Use a quantized dataset for testing operators where a single-bit error is expected. With a float dataset, the error depended on the output quantization. With a quantized dataset, the tolerance is always `1`, regardless of quantization. ### Test plan Tested by existing nxp tests. `./backends/nxp/run_unittests.sh` cc @robert-kalmar @JakeStevens @digantdesai @rascani
1 parent f2ab082 commit 9ed0238

6 files changed

Lines changed: 59 additions & 49 deletions

File tree

‎backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape):
5656
mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={}
5757
)
5858
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
59+
60+
# Quantize the dataset and allow a single bit error.
61+
remove_quant_io_ops = True
5962
comparator = AllCloseOutputComparator(atol=1)
6063

6164
lower_run_compare(
@@ -65,7 +68,7 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape):
6568
request,
6669
dataset_creator,
6770
comparator,
68-
remove_quant_io_ops=True,
71+
remove_quant_io_ops=remove_quant_io_ops,
6972
)
7073

7174
def test__basic_nsys_inference_qat(self, mocker, request):
@@ -75,6 +78,9 @@ def test__basic_nsys_inference_qat(self, mocker, request):
7578
mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={}
7679
)
7780
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
81+
82+
# Quantize the dataset and allow a single bit error.
83+
remove_quant_io_ops = True
7884
comparator = AllCloseOutputComparator(atol=1)
7985

8086
lower_run_compare(
@@ -84,7 +90,7 @@ def test__basic_nsys_inference_qat(self, mocker, request):
8490
request,
8591
dataset_creator,
8692
comparator,
87-
remove_quant_io_ops=True,
93+
remove_quant_io_ops=remove_quant_io_ops,
8894
use_qat=True,
8995
)
9096

@@ -117,6 +123,9 @@ def test__broadcast(self, mocker, request, input_spec):
117123
mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={}
118124
)
119125
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
126+
127+
# Quantize the dataset and allow a single bit error.
128+
remove_quant_io_ops = True
120129
comparator = AllCloseOutputComparator(atol=1)
121130

122131
lower_run_compare(
@@ -126,7 +135,7 @@ def test__broadcast(self, mocker, request, input_spec):
126135
request,
127136
dataset_creator,
128137
comparator,
129-
remove_quant_io_ops=True,
138+
remove_quant_io_ops=remove_quant_io_ops,
130139
)
131140

132141
@pytest.mark.parametrize(
@@ -230,6 +239,9 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec):
230239
expected_non_delegated_ops={},
231240
)
232241
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
242+
243+
# Quantize the dataset and allow a single bit error.
244+
remove_quant_io_ops = True
233245
comparator = AllCloseOutputComparator(atol=1)
234246

235247
lower_run_compare(
@@ -239,5 +251,5 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec):
239251
request,
240252
dataset_creator,
241253
comparator,
242-
remove_quant_io_ops=True,
254+
remove_quant_io_ops=remove_quant_io_ops,
243255
)

‎backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,9 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec):
204204
expected_non_delegated_ops={},
205205
)
206206
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
207+
208+
# Quantize the dataset and allow a single bit error.
209+
remove_quant_io_ops = True
207210
comparator = AllCloseOutputComparator(atol=1)
208211

209212
lower_run_compare(
@@ -213,5 +216,5 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec):
213216
request,
214217
dataset_creator,
215218
comparator,
216-
remove_quant_io_ops=True,
219+
remove_quant_io_ops=remove_quant_io_ops,
217220
)

‎backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,13 @@
1010
import pytest
1111
import torch
1212

13-
from executorch.backends.nxp.neutron_partitioner import NeutronPartitioner
1413
from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator
1514
from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier
1615
from executorch.backends.nxp.tests.model_output_comparator import (
1716
AllCloseOutputComparator,
1817
)
1918
from executorch.backends.nxp.tests.nsys_testing import lower_run_compare
20-
from executorch.backends.nxp.tests.ops_aliases import DequantizePerTensor, Sigmoid
19+
from executorch.backends.nxp.tests.ops_aliases import Sigmoid
2120
from torch import nn
2221
from executorch.backends.nxp.tests.use_qat import * # noqa F403
2322

@@ -30,9 +29,7 @@ def reseed_model_per_test_run():
3029

3130
class TestSigmoid:
3231
# noinspection PyMethodMayBeStatic
33-
def assert_delegated(
34-
self, model, input_shape, mocker, request, use_qat=False, atol=None
35-
):
32+
def assert_delegated(self, model, input_shape, mocker, request, use_qat=False):
3633
graph_verifier = DetailedGraphVerifier(
3734
mocker,
3835
expected_delegated_ops={Sigmoid: 1},
@@ -42,17 +39,19 @@ def assert_delegated(
4239
# Create a RandomDatasetCreator that covers also negative numbers to properly test the operator.
4340
dataset_creator = RandomDatasetCreator(low=-2, high=2)
4441

45-
kwargs = {"atol": atol} if atol is not None else {}
46-
output_comparator = AllCloseOutputComparator(**kwargs)
42+
# Quantize the dataset and allow a single bit error.
43+
remove_quant_io_ops = True
44+
comparator = AllCloseOutputComparator(atol=1)
4745

4846
lower_run_compare(
4947
model,
5048
input_shape,
5149
graph_verifier,
5250
request,
5351
dataset_creator,
54-
output_comparator,
52+
comparator,
5553
use_qat=use_qat,
54+
remove_quant_io_ops=remove_quant_io_ops,
5655
)
5756

5857
def test__basic_nsys_inference__qat(self, mocker, request, use_qat):
@@ -74,14 +73,4 @@ def test__basic_nsys_inference__qat(self, mocker, request, use_qat):
7473
def test__input_shapes(self, mocker, request, input_shape):
7574
model = nn.Sigmoid()
7675

77-
output_scale = 1.0 / 256.0
78-
lowering_spy = mocker.spy(NeutronPartitioner, "partition")
79-
self.assert_delegated(
80-
model, input_shape, mocker, request, atol=output_scale
81-
) # Allow single bit error.
82-
83-
# Verify that the `atol` is indeed equal to the output scale.
84-
# In the near future, we would like to add support for testing with int8 IO, where this check will be trivial.
85-
nodes = list(lowering_spy.spy_return.tagged_exported_program.graph.nodes)
86-
assert nodes[-2].target == DequantizePerTensor
87-
assert nodes[-2].args[1] == output_scale
76+
self.assert_delegated(model, input_shape, mocker, request)

‎backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape):
5656
mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={}
5757
)
5858
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
59+
60+
# Quantize the dataset and allow a single bit error.
61+
remove_quant_io_ops = True
5962
comparator = AllCloseOutputComparator(atol=1)
6063

6164
lower_run_compare(
@@ -65,7 +68,7 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape):
6568
request,
6669
dataset_creator,
6770
comparator,
68-
remove_quant_io_ops=True,
71+
remove_quant_io_ops=remove_quant_io_ops,
6972
)
7073

7174
def test__basic_nsys_inference_qat(self, mocker, request):
@@ -75,6 +78,9 @@ def test__basic_nsys_inference_qat(self, mocker, request):
7578
mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={}
7679
)
7780
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
81+
82+
# Quantize the dataset and allow a single bit error.
83+
remove_quant_io_ops = True
7884
comparator = AllCloseOutputComparator(atol=1)
7985

8086
lower_run_compare(
@@ -85,7 +91,7 @@ def test__basic_nsys_inference_qat(self, mocker, request):
8591
dataset_creator,
8692
comparator,
8793
use_qat=True,
88-
remove_quant_io_ops=True,
94+
remove_quant_io_ops=remove_quant_io_ops,
8995
)
9096

9197
@pytest.mark.parametrize(
@@ -113,6 +119,9 @@ def test__broadcast(self, mocker, request, input_spec):
113119
mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={}
114120
)
115121
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
122+
123+
# Quantize the dataset and allow a single bit error.
124+
remove_quant_io_ops = True
116125
comparator = AllCloseOutputComparator(atol=1)
117126

118127
lower_run_compare(
@@ -122,7 +131,7 @@ def test__broadcast(self, mocker, request, input_spec):
122131
request,
123132
dataset_creator,
124133
comparator,
125-
remove_quant_io_ops=True,
134+
remove_quant_io_ops=remove_quant_io_ops,
126135
)
127136

128137
@pytest.mark.parametrize(
@@ -225,6 +234,9 @@ def test__broadcast_channels_first_input(self, mocker, request, input_spec):
225234
expected_non_delegated_ops={},
226235
)
227236
dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
237+
238+
# Quantize the dataset and allow a single bit error.
239+
remove_quant_io_ops = True
228240
comparator = AllCloseOutputComparator(atol=1)
229241

230242
lower_run_compare(
@@ -234,5 +246,5 @@ def test__broadcast_channels_first_input(self, mocker, request, input_spec):
234246
request,
235247
dataset_creator,
236248
comparator,
237-
remove_quant_io_ops=True,
249+
remove_quant_io_ops=remove_quant_io_ops,
238250
)

‎backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ def forward(self, x):
5151

5252

5353
class TestUpsampleBilinear2D:
54-
# TODO Use quantized dataset and `atol=1` in the tests.
5554

5655
# noinspection PyMethodMayBeStatic
5756
def assert_delegated(
@@ -61,7 +60,6 @@ def assert_delegated(
6160
mocker,
6261
request,
6362
use_qat=False,
64-
atol=None,
6563
expected_delegated_ops=None,
6664
):
6765
if expected_delegated_ops is None:
@@ -76,17 +74,19 @@ def assert_delegated(
7674
# Cover also negative values to thoroughly test the operator.
7775
dataset_creator = RandomDatasetCreator(low=-2, high=2)
7876

79-
kwargs = {"atol": atol} if atol is not None else {}
80-
output_comparator = AllCloseOutputComparator(**kwargs)
77+
# Quantize the dataset and allow a single bit error.
78+
remove_quant_io_ops = True
79+
comparator = AllCloseOutputComparator(atol=1)
8180

8281
lower_run_compare(
8382
model,
8483
input_shape,
8584
graph_verifier,
8685
request,
8786
dataset_creator,
88-
output_comparator,
87+
comparator,
8988
use_qat=use_qat,
89+
remove_quant_io_ops=remove_quant_io_ops,
9090
)
9191

9292
# noinspection PyMethodMayBeStatic
@@ -103,20 +103,14 @@ def test__qat__align_corners(self, mocker, request, use_qat):
103103
input_shape = (1, 2, 3, 4)
104104
output_size = (5, 7)
105105
model = UpsampleBilinearModule(size=output_size, align_corners=align_corners)
106-
atol = 0.015 # ~= output scale -> single bit error.
107-
self.assert_delegated(
108-
model, input_shape, mocker, request, use_qat=use_qat, atol=atol
109-
)
106+
self.assert_delegated(model, input_shape, mocker, request, use_qat=use_qat)
110107

111108
def test__qat__not_align_corners(self, mocker, request, use_qat):
112109
align_corners = False
113110
input_shape = (1, 2, 3, 4)
114111
output_size = (6, 8)
115112
model = UpsampleBilinearModule(size=output_size, align_corners=align_corners)
116-
atol = 0.015 # ~= output scale -> single bit error.
117-
self.assert_delegated(
118-
model, input_shape, mocker, request, use_qat=use_qat, atol=atol
119-
)
113+
self.assert_delegated(model, input_shape, mocker, request, use_qat=use_qat)
120114

121115
@pytest.mark.parametrize(
122116
"input_shape, output_size",
@@ -136,8 +130,7 @@ def test__not_align_corners__output_size(
136130
):
137131
align_corners = False
138132
model = UpsampleBilinearModule(size=output_size, align_corners=align_corners)
139-
atol = 0.016 # ~= output scale -> single bit error.
140-
self.assert_delegated(model, input_shape, mocker, request, atol=atol)
133+
self.assert_delegated(model, input_shape, mocker, request)
141134

142135
def test__not_align_corners__output_size__unsupported(self):
143136
align_corners = False
@@ -162,8 +155,7 @@ def test__not_align_corners__output_size__unsupported(self):
162155
def test__not_align_corners__scales(self, mocker, request, input_shape, scale):
163156
align_corners = False
164157
model = UpsampleBilinearModule(scale=scale, align_corners=align_corners)
165-
atol = 0.016 # ~= output scale -> single bit error.
166-
self.assert_delegated(model, input_shape, mocker, request, atol=atol)
158+
self.assert_delegated(model, input_shape, mocker, request)
167159

168160
def test__not_align_corners__scales__unsupported(self):
169161
align_corners = False
@@ -196,8 +188,7 @@ def test__align_corners__output_size(
196188
):
197189
align_corners = True
198190
model = UpsampleBilinearModule(size=output_size, align_corners=align_corners)
199-
atol = 0.016 # ~= output scale -> single bit error.
200-
self.assert_delegated(model, input_shape, mocker, request, atol=atol)
191+
self.assert_delegated(model, input_shape, mocker, request)
201192

202193
def test__align_corners__output_size__unsupported(self):
203194
align_corners = True
@@ -253,8 +244,7 @@ def test__align_corners__output_size__input_size_equal_to_one(self):
253244
def test__align_corners__scales(self, mocker, request, input_shape, scale):
254245
align_corners = True
255246
model = UpsampleBilinearModule(scale=scale, align_corners=align_corners)
256-
atol = 0.016 # ~= output scale -> single bit error.
257-
self.assert_delegated(model, input_shape, mocker, request, atol=atol)
247+
self.assert_delegated(model, input_shape, mocker, request)
258248

259249
def test__align_corners__scales__unsupported(self):
260250
align_corners = True

‎backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,9 @@ def assert_delegated_and_correct(
177177
)
178178

179179
dataset = RandomDatasetCreator(low=-128, high=128)
180+
181+
# Quantize the dataset and allow a single bit error.
182+
remove_quant_io_ops = True
180183
comparator = AllCloseOutputComparator(atol=1)
181184

182185
lower_run_compare(
@@ -188,6 +191,7 @@ def assert_delegated_and_correct(
188191
comparator,
189192
mocker=mocker,
190193
use_qat=use_qat,
194+
remove_quant_io_ops=remove_quant_io_ops,
191195
)
192196

193197
@staticmethod

0 commit comments

Comments
 (0)