Skip to content

Commit 7d9008a

Browse files
FrancescAltedFrancescAlted
committed
Add benchmarks for reductions, and plots for different archs
1 parent 8d33de5 commit 7d9008a

File tree

3 files changed

+314
-0
lines changed

3 files changed

+314
-0
lines changed

‎bench/ndarray/jit-reduc-float64-plot.py

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#######################################################################
2+
# Copyright (c) 2019-present, Blosc Development Team <[email protected]>
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under a BSD-style license (found in the
6+
# LICENSE file in the root directory of this source tree)
7+
#######################################################################
8+
9+
# Plots for the jit vs. numpy benchmarks on different array sizes and platforms.
10+
11+
import matplotlib.pyplot as plt
12+
import plotly.graph_objects as go
13+
14+
plotly = True
15+
16+
sizes = [1, 5, 10, 20, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110, 115, 120]
17+
sizes_GB = [n * 1000 * n * 1000 * 8 * 2 / 2**30 for n in sizes]
18+
19+
intel = False
20+
amd = True
21+
m2linux = False
22+
23+
# Load the data
24+
if amd:
25+
title_ = "AMD Ryzen 9 9800X3D (64 GB RAM)"
26+
title_ = "np.sum(((a ** 3 + np.sin(a * 2)) < c) & (b > 0), axis=1)"
27+
28+
create_clevel0 = [ 0.0325, 0.2709, 1.0339, 4.0489, 9.0849, 12.4154, 16.7818, 25.5946, 47.5691, 35.9919, 45.4295, 93.3075, 66.6529 ]
29+
compute_clevel0 = [ 0.0017, 0.0243, 0.0869, 0.3370, 0.7665, 1.0375, 1.3727, 1.7377, 2.1472, 2.6205, 3.0435, 18.5878, 28.0816 ]
30+
31+
create_clevel0_ooc = [ 0.0305, 0.3371, 1.3249, 5.0602, 11.0410, 16.3685, 22.2012, 27.1348, 31.7409, 38.0690, 47.4424, 56.9335, 62.6965 ]
32+
compute_clevel0_ooc = [ 0.0019, 0.0243, 0.0885, 0.3434, 0.7761, 1.0724, 1.4082, 1.7373, 2.1827, 2.6124, 7.0940, 9.0734, 10.1089 ]
33+
34+
create_LZ4_l1 = [ 0.0304, 0.2582, 1.0298, 3.9502, 8.8945, 11.9267, 16.3965, 20.2368, 24.6837, 29.3425, 36.2631, 42.1709, 48.0605, 52.3962, 61.5175, 68.6328, 80.1160, 85.4322, 97.1122, 106.9973, 114.8584 ]
35+
compute_LZ4_l1 = [ 0.0018, 0.0210, 0.0756, 0.3003, 0.6609, 0.8886, 1.1285, 1.4453, 1.7959, 2.1889, 2.6978, 3.1586, 3.4286, 3.9929, 4.4590, 5.3601, 5.6702, 6.4690, 6.9764, 7.8714, 8.6404 ]
36+
37+
create_LZ4_l1_ooc = [ 1.7980, 0.2617, 1.0480, 4.0809, 9.0720, 13.8294, 16.7269, 20.5108, 24.9465, 30.0428, 37.1903, 42.8075, 48.7775, 52.9890, 63.4071, 70.1766, 81.9747, 88.1830, 97.7921, 111.0611, 119.7673 ]
38+
compute_LZ4_l1_ooc = [ 0.0019, 0.0214, 0.0795, 0.3060, 0.6985, 0.9195, 1.1766, 1.5213, 1.8845, 2.2972, 2.8044, 3.2587, 3.5898, 4.1524, 4.6293, 5.5485, 5.8715, 6.7386, 7.3019, 8.2307, 9.0145 ]
39+
40+
create_ZSTD_l1 = [ 0.0302, 0.2704, 1.0703, 4.1243, 9.2185, 12.5026, 17.0585, 20.8708, 25.5844, 31.0571, 37.7114, 42.8297, 50.2696, 54.5773, 63.6311, 73.0370, 84.0092, 89.0686, 100.3300, 108.8173, 119.1154 ]
41+
compute_ZSTD_l1 = [ 0.0021, 0.0296, 0.1045, 0.3979, 0.8787, 1.3064, 1.7404, 2.1938, 2.6780, 3.3929, 3.8601, 4.3665, 5.0127, 5.7346, 6.1056, 7.9448, 8.2872, 9.4659, 9.2376, 10.4273, 11.6572 ]
42+
43+
create_ZSTD_l1_ooc = [ 0.6564, 0.2825, 1.0826, 4.1968, 9.5022, 13.4840, 17.5387, 21.5807, 26.0052, 31.3524, 38.5889, 44.1105, 49.8849, 55.5297, 64.6479, 72.7471, 84.6595, 90.4970, 99.9710, 111.6817, 120.8941 ]
44+
compute_ZSTD_l1_ooc = [ 0.0022, 0.0300, 0.1066, 0.4099, 0.8974, 1.3218, 1.7679, 2.2154, 2.7007, 3.4267, 3.9255, 4.4597, 5.1155, 5.8251, 6.2064, 8.0141, 8.4316, 9.3195, 9.4570, 10.7034, 11.9192 ]
45+
46+
create_numpy = [ 0.0020, 0.0527, 0.2292, 0.9412, 2.1043, 2.8286, 3.7046, 4.7217, 5.8308, 7.0491 ]
47+
compute_numpy = [ 0.0179, 0.2495, 0.9840, 3.9263, 8.8450, 12.0259, 16.3507, 40.1672, 155.1292, 302.5115 ]
48+
49+
create_numpy_jit = [ 0.0019, 0.0529, 0.2261, 0.9219, 2.0589, 2.8350, 3.7131, 18.4375, 26.5959, 34.5221, 33.7157, 49.6762, 63.1401 ]
50+
compute_numpy_jit = [ 0.0035, 0.0180, 0.0622, 0.2307, 0.5196, 0.7095, 0.9251, 1.1981, 1.4729, 2.2007, 2.0953, 12.6746, 26.6424 ]
51+
52+
elif intel:
53+
title_ = "Intel Core i9-13900K (32 GB RAM)"
54+
create_clevel0 = [ 0.1810, 0.3511, 1.1511, 4.4575, 10.3164, 17.4344, 24.4274, 37.7116, 36.6179, 53.7264 ]
55+
compute_clevel0 = [ 0.0045, 0.0133, 0.0506, 0.2086, 0.4603, 0.8689, 1.1458, 1.4150, 1.7656, 1.9475 ]
56+
57+
create_LZ4_l1 = [ 0.1834, 0.3457, 1.1234, 4.3301, 10.0406, 16.9509, 22.1617, 26.3818, 32.4472, 39.3830, 41.9484, 52.6316 ]
58+
compute_LZ4_l1 = [ 0.0014, 0.0128, 0.0494, 0.1958, 0.4387, 0.8207, 1.0208, 1.2739, 1.5062, 1.7446, 2.1553, 2.4458 ]
59+
60+
create_ZSTD_l1 = [ 0.0362, 0.3734, 1.2009, 4.5362, 10.3706, 18.7104, 23.1148, 27.6572, 33.7207, 41.0326, 44.2322, 54.9467 ]
61+
compute_ZSTD_l1 = [ 0.0028, 0.0193, 0.0799, 0.2226, 0.4983, 0.9072, 1.1624, 1.4375, 1.8162, 2.0918, 2.5067, 2.7760 ]
62+
63+
create_numpy = [ 0.0046, 0.1160, 0.4327, 1.7166, 3.8661, 7.5005, 14.1090, 18.6720, 64.2425, 108.4532, 529.1393, 962.1662 ]
64+
compute_numpy = [ 0.0240, 0.1920, 0.7217, 2.9316, 6.5893, 14.0353, 47.4275, 99.0893, 187.8040, 202.3973, 460.5915, 551.2776 ]
65+
66+
elif m2linux:
67+
title_ = "MacBook Air M2 (24 GB RAM)"
68+
create_LZ4_l1 = [0.021555185317993164, 0.2862977981567383, 0.8696625232696533, 3.4979920387268066, 8.235799789428711, 13.708781242370605, 20.74394702911377, 33.23137378692627]
69+
compute_LZ4_l1 = [0.0033464431762695312, 0.03627762794494629, 0.14009513854980468, 0.5438736915588379, 1.2493964672088622, 2.194223642349243, 3.5851136207580567, 5.067658472061157]
70+
71+
create_numpy = [0.0016903877258300781, 0.04910874366760254, 0.18264532089233398, 0.7124006748199463, 1.8350563049316406, 8.877023935317993, 101.2457287311554, 196.21723294258118]
72+
compute_numpy = [0.003887462615966797, 0.026979732513427734, 0.11047358512878418, 0.4213367462158203, 0.9288184165954589, 1.6470709323883057, 5.5601390361785885, 9.401740503311157]
73+
74+
else:
75+
title_ = "Mac Mini M4 Pro (24 GB RAM)"
76+
77+
create_numpy = [ 0.0024, 0.0686, 0.2857, 1.1800, 3.5006, 13.7092, 21.9491, 30.5237, 101.3553, 363.5005, 446.5876, 1509.1826 ]
78+
compute_numpy = [ 0.0046, 0.1173, 0.5066, 2.0908, 5.6268, 13.0679, 16.7926, 20.9192, 25.5899, 34.5382, 46.0664, 1083.7046 ]
79+
80+
create_numpy_jit = [ 0.0024, 0.0686, 0.2857, 1.1800, 3.5006, 13.7092, 21.9491, 30.5237, 101.3553, 363.5005, 446.5876, 1509.1826 ]
81+
compute_numpy_jit = [ ]
82+
83+
create_clevel0 = [ 0.0321, 0.6742, 1.6750, 5.1412, 12.4538, 20.6695, 28.0185, 34.7422, 41.3935, 50.4275, 59.5572, 71.3740 ]
84+
compute_clevel0 = [ 0.0017, 0.0224, 0.0761, 0.2439, 0.5827, 0.9816, 1.2265, 6.8712, 9.6391, 11.2875, 13.1953, 15.4047 ]
85+
86+
create_LZ4_l1 = [ 0.0316, 0.6974, 1.6957, 5.2077, 11.8975, 20.2346, 26.6419, 32.7686, 38.2088, 47.5221, 55.7224, 66.6138 ]
87+
compute_LZ4_l1 = [ 0.0019, 0.0234, 0.0763, 0.2421, 0.5533, 0.9384, 1.2476, 1.5299, 1.8564, 2.1836, 2.5633, 2.9245 ]
88+
89+
create_ZSTD_l1 = [ 0.0354, 0.7105, 1.7217, 5.2663, 12.7385, 20.3693, 28.1353, 33.0310, 40.3843, 50.2020, 58.0643, 69.5190 ]
90+
compute_ZSTD_l1 = [ 0.0021, 0.0225, 0.0773, 0.2630, 0.8735, 1.0087, 1.9553, 1.5670, 3.1986, 3.3499, 4.0728, 4.6287 ]
91+
92+
93+
# Plot the data. There will be 2 plots: one for create times and another for compute times
94+
labels = ['lvl=0', 'LZ4 lvl=1', 'ZSTD lvl=1', 'NumPy', "numpy_jit"]
95+
if plotly:
96+
# Create the create times plot
97+
fig_create = go.Figure()
98+
fig_create.add_trace(
99+
go.Scatter(x=sizes_GB, y=create_clevel0, mode='lines+markers', name=labels[0]))
100+
fig_create.add_trace(
101+
go.Scatter(x=sizes_GB, y=create_clevel0_ooc, mode='lines+markers', name=labels[0] + "(ooc)"))
102+
fig_create.add_trace(
103+
go.Scatter(x=sizes_GB, y=create_LZ4_l1, mode='lines+markers', name=labels[1]))
104+
fig_create.add_trace(
105+
go.Scatter(x=sizes_GB, y=create_LZ4_l1_ooc, mode='lines+markers', name=labels[1] + "(ooc)"))
106+
fig_create.add_trace(
107+
go.Scatter(x=sizes_GB, y=create_ZSTD_l1, mode='lines+markers', name=labels[2]))
108+
fig_create.add_trace(
109+
go.Scatter(x=sizes_GB, y=create_ZSTD_l1_ooc, mode='lines+markers', name=labels[2] + "(ooc)"))
110+
fig_create.add_trace(
111+
go.Scatter(x=sizes_GB, y=create_numpy_jit, mode='lines+markers', name=labels[3]))
112+
fig_create.update_layout(title=f'Create operands times ({title_})', xaxis_title='Size (GB)', yaxis_title='Time (s)')
113+
114+
# Create the compute times plot
115+
# Calculate the maximum y1 value
116+
y1_max = max(max(compute_clevel0), max(compute_clevel0_ooc), max(compute_LZ4_l1), max(compute_LZ4_l1_ooc),
117+
max(compute_ZSTD_l1), max(compute_ZSTD_l1_ooc), max(compute_numpy), max(compute_numpy_jit))
118+
119+
fig_compute = go.Figure()
120+
fig_compute.add_trace(
121+
go.Scatter(x=sizes_GB, y=compute_clevel0, mode='lines+markers', name=labels[0]))
122+
fig_compute.add_trace(
123+
go.Scatter(x=sizes_GB, y=compute_clevel0_ooc, mode='lines+markers', name=labels[0] + "(ooc)"))
124+
fig_compute.add_trace(
125+
go.Scatter(x=sizes_GB, y=compute_LZ4_l1, mode='lines+markers', name=labels[1]))
126+
fig_compute.add_trace(
127+
go.Scatter(x=sizes_GB, y=compute_LZ4_l1_ooc, mode='lines+markers', name=labels[1] + "(ooc)"))
128+
fig_compute.add_trace(
129+
go.Scatter(x=sizes_GB, y=compute_ZSTD_l1, mode='lines+markers', name=labels[2]))
130+
fig_compute.add_trace(
131+
go.Scatter(x=sizes_GB, y=compute_ZSTD_l1_ooc, mode='lines+markers', name=labels[2] + "(ooc)"))
132+
fig_compute.add_trace(go.Scatter(x=sizes_GB, y=compute_numpy, mode='lines+markers', name=labels[3]))
133+
fig_compute.add_trace(go.Scatter(x=sizes_GB, y=compute_numpy_jit, mode='lines+markers', name=labels[4]))
134+
fig_compute.update_layout(title=f'Compute times ({title_})', xaxis_title='Size (GB)', yaxis_title='Time (s)')
135+
136+
# Add a vertical line at 64 GB
137+
y1_max = 35
138+
fig_compute.add_shape(
139+
type="line", x0=64, y0=0, x1=64, y1=y1_max,
140+
line=dict(color="Gray", width=2, dash="dot"),
141+
)
142+
if amd:
143+
fig_compute.add_annotation(x=64, y=y1_max * .9, text="64 GB", showarrow=True, arrowhead=2, ax=40, ay=0)
144+
145+
# Show the plots
146+
fig_create.show()
147+
fig_compute.show()
148+
else:
149+
plt.figure()
150+
plt.plot(sizes_GB, create_clevel0, "o-", label=labels[0])
151+
plt.plot(sizes_GB, create_LZ4_l1, "o-", label=labels[1])
152+
plt.plot(sizes_GB, create_ZSTD_l1, "o-", label=labels[2])
153+
plt.plot(sizes_GB, create_numpy_jit, "o-", label=labels[3])
154+
plt.xlabel("Size (GB)")
155+
plt.ylabel("Time (s)")
156+
plt.title(f"Create times ({title_})")
157+
plt.legend()
158+
# Now, the compute times
159+
plt.figure()
160+
plt.plot(sizes_GB, compute_clevel0, "o-", label=labels[0])
161+
plt.plot(sizes_GB, compute_LZ4_l1, "o-", label=labels[1])
162+
plt.plot(sizes_GB, compute_ZSTD_l1, "o-", label=labels[2])
163+
plt.plot(sizes_GB, compute_numpy, "o-", label=labels[3])
164+
plt.plot(sizes_GB, compute_numpy_jit, "o-", label=labels[4])
165+
plt.xlabel("Size (GB)")
166+
plt.ylabel("Time (s)")
167+
plt.title(f"Compute times ({title_})")
168+
plt.legend()
169+
plt.show()

‎bench/ndarray/jit-reduc-sizes.py

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#######################################################################
2+
# Copyright (c) 2019-present, Blosc Development Team <[email protected]>
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under a BSD-style license (found in the
6+
# LICENSE file in the root directory of this source tree)
7+
#######################################################################
8+
9+
# Compute reductions for different array sizes, using the jit decorator
10+
# and different operands (NumPy and NDArray). Different compression
11+
# levels and codecs can be selected. For in-memory testing, set ooc to False.
12+
13+
from time import time
14+
import blosc2
15+
import numpy as np
16+
import sys
17+
18+
niter = 5
19+
#dtype = np.dtype("float32")
20+
dtype = np.dtype("float64")
21+
clevel = 1
22+
numpy = False
23+
numpy_jit = False
24+
ooc = True # for in-memory testing, set to False
25+
26+
cparams = cparams_out = None
27+
28+
# For 64 GB RAM
29+
# sizes_numpy = (1, 5, 10, 20, 30, 35, 40, 45, 50, 55)
30+
# sizes_numpy_jit = (1, 5, 10, 20, 30, 35, 40, 45, 50, 55, 60, 65, 70)
31+
# sizes_clevel0 = (1, 5, 10, 20, 30, 35, 40, 45, 50, 55, 60, 65, 70)
32+
# size_list = (1, 5, 10, 20, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110) # limit clevel>=1 float64
33+
34+
# For 24 GB RAM
35+
sizes_numpy = (1, 5, 10, 20, 30, 35, 40) # limit numpy float64
36+
sizes_numpy_jit = (1, 5, 10, 20, 30, 35, 40, 45) # limit numpy float64
37+
sizes_clevel0 = (1, 5, 10, 20, 30, 35, 40, 45) # limit clevel==0 float64
38+
size_list = (1, 5, 10, 20, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90) # limit clevel>=1 float64
39+
40+
codec = "LZ4" # default codec
41+
if len(sys.argv) > 2:
42+
codec = sys.argv[2]
43+
if len(sys.argv) > 1:
44+
try:
45+
clevel = int(sys.argv[1])
46+
except ValueError:
47+
if sys.argv[1] == "numpy":
48+
numpy = True
49+
elif sys.argv[1] == "numpy_jit":
50+
numpy = True
51+
numpy_jit = True
52+
else:
53+
raise ValueError("Invalid argument")
54+
55+
apath = bpath = None
56+
if numpy:
57+
print("Using NumPy arrays as operands")
58+
else:
59+
print("Using NDArray arrays as operands")
60+
cparams = cparams_out = blosc2.CParams(clevel=clevel, codec=blosc2.Codec[codec])
61+
# cparams_out = blosc2.CParams(clevel=clevel, codec=blosc2.Codec.LZ4)
62+
print("Using cparams: ", cparams)
63+
if ooc:
64+
apath = "a.b2nd"
65+
bpath = "b.b2nd"
66+
67+
68+
# The reductions to compute
69+
def compute_reduction_numpy(a, b, c):
70+
return np.sum(((a ** 3 + np.sin(a * 2)) < c) & (b > 0), axis=1)
71+
72+
@blosc2.jit
73+
def compute_reduction(a, b, c):
74+
return np.sum(((a ** 3 + np.sin(a * 2)) < c) & (b > 0), axis=1)
75+
76+
77+
create_times = []
78+
compute_times = []
79+
for n in size_list:
80+
if clevel == 0 and n not in sizes_clevel0:
81+
break
82+
if numpy_jit and n not in sizes_numpy_jit:
83+
break
84+
if numpy and not numpy_jit and n not in sizes_numpy:
85+
break
86+
N = n * 1000
87+
print(f"\nN = {n}000, {dtype=}, size={N ** 2 * 2 * dtype.itemsize / 2**30 }")
88+
chunks = (100, N)
89+
blocks = (1, N)
90+
chunks, blocks = None, None # automatic chunk and block sizes
91+
# Lossy compression
92+
#filters = [blosc2.Filter.TRUNC_PREC, blosc2.Filter.SHUFFLE]
93+
#filters_meta = [8, 0] # keep 8 bits of precision in mantissa
94+
#cparams = blosc2.CParams(clevel=1, codec=blosc2.Codec.LZ4, filters=filters, filters_meta=filters_meta)
95+
96+
# Create some data operands
97+
t0 = time()
98+
if numpy:
99+
a = np.linspace(0, 1, N * N, dtype=dtype).reshape(N, N)
100+
b = np.linspace(1, 2, N * N, dtype=dtype).reshape(N, N)
101+
#b = a + 1
102+
c = np.linspace(-10, 10, N, dtype=dtype)
103+
else:
104+
a = blosc2.linspace(0, 1, N * N, dtype=dtype, shape=(N, N), cparams=cparams, urlpath=apath, mode="w")
105+
print("a.chunks, a.blocks, a.schunk.cratio: ", a.chunks, a.blocks, a.schunk.cratio)
106+
b = blosc2.linspace(1, 2, N * N, dtype=dtype, shape=(N, N), cparams=cparams, urlpath=bpath, mode="w")
107+
#b = (a + 1).compute(cparams=cparams, chunks=chunks, blocks=blocks)
108+
#print(b.chunks, b.blocks, b.schunk.cratio, b.cparams)
109+
c = blosc2.linspace(-10, 10, N, dtype=dtype, cparams=cparams) # broadcasting is supported
110+
#c = blosc2.linspace(-10, 10, N * N, dtype=dtype, shape=(N, N), cparams=cparams)
111+
t1 = time() - t0
112+
print(f"Time to create data: {t1:.4f}")
113+
create_times.append(t1)
114+
115+
if numpy:
116+
if numpy_jit:
117+
out = compute_reduction(a, b, c)
118+
t0 = time()
119+
for i in range(niter):
120+
out = compute_reduction(a, b, c)
121+
t1 = (time() - t0) / niter
122+
print(f"Time to compute with numpy_jit and NumPy operands: {t1:.4f}")
123+
else:
124+
t0 = time()
125+
nout = compute_reduction_numpy(a, b, c)
126+
t1 = time() - t0
127+
print(f"Time to compute with NumPy engine: {t1:.4f}")
128+
else:
129+
out = compute_reduction(a, b, c)
130+
t0 = time()
131+
for i in range(niter):
132+
out = compute_reduction(a, b, c)
133+
t1 = (time() - t0) / niter
134+
print(f"Time to compute with numpy_jit and {clevel=}: {t1:.4f}")
135+
compute_times.append(t1)
136+
del a, b, c
137+
138+
print("\nCreate times: [", ", ".join([f"{t:.4f}" for t in create_times]), "]")
139+
print("Compute times: [", ", ".join([f"{t:.4f}" for t in compute_times]), "]")
140+
print("End of run!\n\n")

‎bench/ndarray/run-jit-reduc-sizes.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
/usr/bin/time -v python bench/ndarray/jit-reduc-sizes.py numpy
2+
/usr/bin/time -v python bench/ndarray/jit-reduc-sizes.py numpy_jit
3+
/usr/bin/time -v python bench/ndarray/jit-reduc-sizes.py 0
4+
/usr/bin/time -v python bench/ndarray/jit-reduc-sizes.py 1 LZ4
5+
/usr/bin/time -v python bench/ndarray/jit-reduc-sizes.py 1 ZSTD

0 commit comments

Comments
 (0)