Skip to content

Commit 1155f42

Browse files
ConorMacBrideConorMacBride
committed
Automatically apply regex and remove specific hashes
1 parent b8bcfe1 commit 1155f42

10 files changed

+295
-241
lines changed

‎tests/subtests/helpers.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,3 +255,36 @@ def apply_regex(file, regex_paths, regex_strs):
255255

256256
with open(file, 'w') as f:
257257
json.dump(summary, f, indent=2)
258+
259+
260+
def remove_specific_hashes(summary_file):
261+
"""Replace all hashes in a summary file with placeholder values.
262+
263+
This is done because the actual hashes used for testing are taken from
264+
separate files for each specific matplotlib version.
265+
"""
266+
267+
baseline_placeholder = "###_BASELINE_HASH_###"
268+
result_placeholder = "###_RESULT_HASH_###"
269+
270+
with open(summary_file, "r") as f:
271+
summary = json.load(f)
272+
273+
for test in summary.keys():
274+
275+
# Get actual hashes
276+
baseline = summary[test]["baseline_hash"]
277+
result = summary[test]["result_hash"]
278+
279+
# Replace with placeholders (if summary has hashes)
280+
if baseline is not None:
281+
summary[test]["baseline_hash"] = baseline_placeholder
282+
summary[test]["status_msg"] = \
283+
summary[test]["status_msg"].replace(baseline, baseline_placeholder)
284+
if result is not None:
285+
summary[test]["result_hash"] = result_placeholder
286+
summary[test]["status_msg"] = \
287+
summary[test]["status_msg"].replace(result, result_placeholder)
288+
289+
with open(summary_file, "w") as f:
290+
json.dump(summary, f, indent=2)

‎tests/subtests/summaries/test_generate.json

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"rms": null,
1010
"tolerance": null,
1111
"result_image": null,
12-
"baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
12+
"baseline_hash": "###_BASELINE_HASH_###",
1313
"result_hash": null
1414
},
1515
"subtests.subtest.test_hmatch_idiff": {
@@ -22,7 +22,7 @@
2222
"rms": null,
2323
"tolerance": null,
2424
"result_image": null,
25-
"baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
25+
"baseline_hash": "###_BASELINE_HASH_###",
2626
"result_hash": null
2727
},
2828
"subtests.subtest.test_hmatch_idiffshape": {
@@ -35,7 +35,7 @@
3535
"rms": null,
3636
"tolerance": null,
3737
"result_image": null,
38-
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
38+
"baseline_hash": "###_BASELINE_HASH_###",
3939
"result_hash": null
4040
},
4141
"subtests.subtest.test_hmatch_imissing": {
@@ -48,7 +48,7 @@
4848
"rms": null,
4949
"tolerance": null,
5050
"result_image": null,
51-
"baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
51+
"baseline_hash": "###_BASELINE_HASH_###",
5252
"result_hash": null
5353
},
5454
"subtests.subtest.test_hdiff_imatch": {
@@ -61,7 +61,7 @@
6161
"rms": null,
6262
"tolerance": null,
6363
"result_image": null,
64-
"baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
64+
"baseline_hash": "###_BASELINE_HASH_###",
6565
"result_hash": null
6666
},
6767
"subtests.subtest.test_hdiff_idiff": {
@@ -74,7 +74,7 @@
7474
"rms": null,
7575
"tolerance": null,
7676
"result_image": null,
77-
"baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
77+
"baseline_hash": "###_BASELINE_HASH_###",
7878
"result_hash": null
7979
},
8080
"subtests.subtest.test_hdiff_idiffshape": {
@@ -87,7 +87,7 @@
8787
"rms": null,
8888
"tolerance": null,
8989
"result_image": null,
90-
"baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
90+
"baseline_hash": "###_BASELINE_HASH_###",
9191
"result_hash": null
9292
},
9393
"subtests.subtest.test_hdiff_imissing": {
@@ -100,7 +100,7 @@
100100
"rms": null,
101101
"tolerance": null,
102102
"result_image": null,
103-
"baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
103+
"baseline_hash": "###_BASELINE_HASH_###",
104104
"result_hash": null
105105
},
106106
"subtests.subtest.test_hmissing_imatch": {
@@ -113,7 +113,7 @@
113113
"rms": null,
114114
"tolerance": null,
115115
"result_image": null,
116-
"baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856",
116+
"baseline_hash": "###_BASELINE_HASH_###",
117117
"result_hash": null
118118
},
119119
"subtests.subtest.test_hmissing_idiff": {
@@ -126,7 +126,7 @@
126126
"rms": null,
127127
"tolerance": null,
128128
"result_image": null,
129-
"baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
129+
"baseline_hash": "###_BASELINE_HASH_###",
130130
"result_hash": null
131131
},
132132
"subtests.subtest.test_hmissing_idiffshape": {
@@ -139,7 +139,7 @@
139139
"rms": null,
140140
"tolerance": null,
141141
"result_image": null,
142-
"baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d",
142+
"baseline_hash": "###_BASELINE_HASH_###",
143143
"result_hash": null
144144
},
145145
"subtests.subtest.test_hmissing_imissing": {
@@ -152,7 +152,7 @@
152152
"rms": null,
153153
"tolerance": null,
154154
"result_image": null,
155-
"baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e",
155+
"baseline_hash": "###_BASELINE_HASH_###",
156156
"result_hash": null
157157
},
158158
"subtests.subtest.test_hdiff_imatch_tolerance": {
@@ -165,7 +165,7 @@
165165
"rms": null,
166166
"tolerance": null,
167167
"result_image": null,
168-
"baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
168+
"baseline_hash": "###_BASELINE_HASH_###",
169169
"result_hash": null
170170
},
171171
"subtests.subtest.test_hdiff_idiff_tolerance": {
@@ -178,7 +178,7 @@
178178
"rms": null,
179179
"tolerance": null,
180180
"result_image": null,
181-
"baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
181+
"baseline_hash": "###_BASELINE_HASH_###",
182182
"result_hash": null
183183
},
184184
"subtests.subtest.test_hdiff_imatch_savefig": {
@@ -191,7 +191,7 @@
191191
"rms": null,
192192
"tolerance": null,
193193
"result_image": null,
194-
"baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc",
194+
"baseline_hash": "###_BASELINE_HASH_###",
195195
"result_hash": null
196196
},
197197
"subtests.subtest.test_hdiff_imatch_style": {
@@ -204,7 +204,7 @@
204204
"rms": null,
205205
"tolerance": null,
206206
"result_image": null,
207-
"baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13",
207+
"baseline_hash": "###_BASELINE_HASH_###",
208208
"result_hash": null
209209
},
210210
"subtests.subtest.test_hdiff_imatch_removetext": {
@@ -217,7 +217,7 @@
217217
"rms": null,
218218
"tolerance": null,
219219
"result_image": null,
220-
"baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a",
220+
"baseline_hash": "###_BASELINE_HASH_###",
221221
"result_hash": null
222222
}
223223
}

‎tests/subtests/summaries/test_generate_hashes_only.json

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"rms": null,
1010
"tolerance": 2,
1111
"result_image": null,
12-
"baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
12+
"baseline_hash": "###_BASELINE_HASH_###",
1313
"result_hash": null
1414
},
1515
"subtests.subtest.test_hmatch_idiff": {
@@ -22,7 +22,7 @@
2222
"rms": 11.100353848213828,
2323
"tolerance": 2,
2424
"result_image": "subtests.subtest.test_hmatch_idiff/result.png",
25-
"baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
25+
"baseline_hash": "###_BASELINE_HASH_###",
2626
"result_hash": null
2727
},
2828
"subtests.subtest.test_hmatch_idiffshape": {
@@ -35,7 +35,7 @@
3535
"rms": null,
3636
"tolerance": null,
3737
"result_image": "subtests.subtest.test_hmatch_idiffshape/result.png",
38-
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
38+
"baseline_hash": "###_BASELINE_HASH_###",
3939
"result_hash": null
4040
},
4141
"subtests.subtest.test_hmatch_imissing": {
@@ -48,7 +48,7 @@
4848
"rms": null,
4949
"tolerance": null,
5050
"result_image": "subtests.subtest.test_hmatch_imissing/result.png",
51-
"baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
51+
"baseline_hash": "###_BASELINE_HASH_###",
5252
"result_hash": null
5353
},
5454
"subtests.subtest.test_hdiff_imatch": {
@@ -61,7 +61,7 @@
6161
"rms": null,
6262
"tolerance": 2,
6363
"result_image": null,
64-
"baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
64+
"baseline_hash": "###_BASELINE_HASH_###",
6565
"result_hash": null
6666
},
6767
"subtests.subtest.test_hdiff_idiff": {
@@ -74,7 +74,7 @@
7474
"rms": 11.182677079602481,
7575
"tolerance": 2,
7676
"result_image": "subtests.subtest.test_hdiff_idiff/result.png",
77-
"baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
77+
"baseline_hash": "###_BASELINE_HASH_###",
7878
"result_hash": null
7979
},
8080
"subtests.subtest.test_hdiff_idiffshape": {
@@ -87,7 +87,7 @@
8787
"rms": null,
8888
"tolerance": null,
8989
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90-
"baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
90+
"baseline_hash": "###_BASELINE_HASH_###",
9191
"result_hash": null
9292
},
9393
"subtests.subtest.test_hdiff_imissing": {
@@ -100,7 +100,7 @@
100100
"rms": null,
101101
"tolerance": null,
102102
"result_image": "subtests.subtest.test_hdiff_imissing/result.png",
103-
"baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
103+
"baseline_hash": "###_BASELINE_HASH_###",
104104
"result_hash": null
105105
},
106106
"subtests.subtest.test_hmissing_imatch": {
@@ -113,7 +113,7 @@
113113
"rms": null,
114114
"tolerance": 2,
115115
"result_image": null,
116-
"baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856",
116+
"baseline_hash": "###_BASELINE_HASH_###",
117117
"result_hash": null
118118
},
119119
"subtests.subtest.test_hmissing_idiff": {
@@ -126,7 +126,7 @@
126126
"rms": 12.12938597648977,
127127
"tolerance": 2,
128128
"result_image": "subtests.subtest.test_hmissing_idiff/result.png",
129-
"baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
129+
"baseline_hash": "###_BASELINE_HASH_###",
130130
"result_hash": null
131131
},
132132
"subtests.subtest.test_hmissing_idiffshape": {
@@ -139,7 +139,7 @@
139139
"rms": null,
140140
"tolerance": null,
141141
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142-
"baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d",
142+
"baseline_hash": "###_BASELINE_HASH_###",
143143
"result_hash": null
144144
},
145145
"subtests.subtest.test_hmissing_imissing": {
@@ -152,7 +152,7 @@
152152
"rms": null,
153153
"tolerance": null,
154154
"result_image": "subtests.subtest.test_hmissing_imissing/result.png",
155-
"baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e",
155+
"baseline_hash": "###_BASELINE_HASH_###",
156156
"result_hash": null
157157
},
158158
"subtests.subtest.test_hdiff_imatch_tolerance": {
@@ -165,7 +165,7 @@
165165
"rms": null,
166166
"tolerance": 200,
167167
"result_image": null,
168-
"baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
168+
"baseline_hash": "###_BASELINE_HASH_###",
169169
"result_hash": null
170170
},
171171
"subtests.subtest.test_hdiff_idiff_tolerance": {
@@ -178,7 +178,7 @@
178178
"rms": 29.260332173249314,
179179
"tolerance": 3,
180180
"result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
181-
"baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
181+
"baseline_hash": "###_BASELINE_HASH_###",
182182
"result_hash": null
183183
},
184184
"subtests.subtest.test_hdiff_imatch_savefig": {
@@ -191,7 +191,7 @@
191191
"rms": null,
192192
"tolerance": 2,
193193
"result_image": null,
194-
"baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc",
194+
"baseline_hash": "###_BASELINE_HASH_###",
195195
"result_hash": null
196196
},
197197
"subtests.subtest.test_hdiff_imatch_style": {
@@ -204,7 +204,7 @@
204204
"rms": null,
205205
"tolerance": 2,
206206
"result_image": null,
207-
"baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13",
207+
"baseline_hash": "###_BASELINE_HASH_###",
208208
"result_hash": null
209209
},
210210
"subtests.subtest.test_hdiff_imatch_removetext": {
@@ -217,7 +217,7 @@
217217
"rms": null,
218218
"tolerance": 2,
219219
"result_image": null,
220-
"baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a",
220+
"baseline_hash": "###_BASELINE_HASH_###",
221221
"result_hash": null
222222
}
223223
}

0 commit comments

Comments
 (0)