Skip to content

Commit 3595309

Browse files
haichangsihaichangsi
authored
Benchmark framework skeleton (#364)
* added a script for automatize running daily benchmarks * added a new target for the shp bench * modified CI and added devcloud-daily script added a script to run all benchmarks modified CI to build and run library and then start benchmarks on a devcloud * created new targets for different compile options created 1 more target for both mhp and shp with compile options: -qopenmp -qopt-streaming-stores=always -xCORE-AVX512 -qopt-zmm-usage=high * added benchmarks directory and fixed naming benchmarks * simplified the code * excluded a part of workflow including daily benchmarks to another file * changed subprocess Popen to run, added a dry_run option formatted commands and changed the way of setting env variables * moved the set command down to make the log more readable * adjusted the script to be run from different named build directories * deleted the second target for both shp and mhp adjusted the benchmark script to use only 1 target * reduced the number of parameters for run_mhp and run_shp added click options to insert common parameters via cli * changes adjusted to meet pep8 standards
1 parent 057d814 commit 3595309

File tree

3 files changed

+279
-0
lines changed

3 files changed

+279
-0
lines changed

‎.github/workflows/daily.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# SPDX-FileCopyrightText: Intel Corporation
2+
#
3+
# SPDX-License-Identifier: BSD-3-Clause
4+
5+
name: Benchmarks
6+
7+
on:
8+
schedule:
9+
- cron '0 0 * * *'
10+
11+
env:
12+
CTEST_OUTPUT_ON_FAILURE: 1
13+
IMPI: [email protected]
14+
MKL: [email protected]
15+
TBB: [email protected]
16+
ICX: [email protected]
17+
18+
jobs:
19+
dailyBench:
20+
runs-on: devcloud
21+
steps:
22+
- uses: actions/checkout@v3
23+
- name: Python setup
24+
uses: actions/setup-python@v4
25+
with:
26+
python-version: '3.x'
27+
cache: 'pip'
28+
- name: Build
29+
run: srun -p pvc scripts/devcloud-build.sh
30+
- name: Test
31+
# mpi does not like the way SLURM sets environment variables
32+
run: srun -p pvc /usr/bin/env -i scripts/devcloud-run.sh
33+
- name: Run benchmarks
34+
run: srun -p pvc /usr/bin/env -i scripts/devcloud-daily.sh

‎benchmarks/runner/run_benchmarks.py

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
#!/usr/bin/env python3
2+
3+
# SPDX-FileCopyrightText: Intel Corporation
4+
#
5+
# SPDX-License-Identifier: BSD-3-Clause
6+
7+
import datetime
8+
import os
9+
import subprocess
10+
11+
import click
12+
13+
14+
def bench_common_params(func):
15+
func = click.option(
16+
'--vec_size', default=1000000, type=int, help='Size of a vector'
17+
)(func)
18+
func = click.option(
19+
'--reps', default=100, type=int, help='Number of reps'
20+
)(func)
21+
func = click.option(
22+
'--bench_filter',
23+
default='Stream_',
24+
type=str,
25+
help='A filter used for a benchmark',
26+
)(func)
27+
return func
28+
29+
30+
@click.command()
31+
@click.option(
32+
'--dry_run', default=False, help='Emits commands but does not execute'
33+
)
34+
def print_run_command(command: str, dry_run: bool):
35+
click.echo(f"Current benchmark command used: \n{command}")
36+
if not dry_run:
37+
subprocess.run(
38+
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
39+
)
40+
41+
42+
@click.command()
43+
@bench_common_params
44+
def run_mhp(
45+
vec_size: int,
46+
reps: int,
47+
bench_filter: str,
48+
sycl_used: bool,
49+
mpirun: bool = True,
50+
n: int = None,
51+
pin_domain: str = '',
52+
pin_order: str = '',
53+
pin_cell: str = '',
54+
):
55+
bench_path = "./benchmarks/gbench/mhp/mhp-bench"
56+
time_now_string = datetime.datetime.now().isoformat(timespec='minutes')
57+
directory_path = "benchmarks/json_output/mhp"
58+
if not os.path.exists(directory_path):
59+
os.makedirs(directory_path)
60+
bench_out = (
61+
f"--benchmark_out={directory_path}"
62+
f"/mhp_benchmark_{time_now_string}.json --benchmark_out_format=json"
63+
)
64+
if n is None:
65+
n = ""
66+
else:
67+
n = f"-n {str(n)}"
68+
if mpirun:
69+
mpirun = "mpirun"
70+
else:
71+
mpirun = ""
72+
n = ""
73+
if sycl_used:
74+
command = (
75+
f"BENCHMARK_FILTER={bench_filter} I_MPI_PIN_DOMAIN={pin_domain} "
76+
f"I_MPI_PIN_ORDER={pin_order} I_MPI_PIN_CELL={pin_cell} {mpirun} "
77+
f"{n} {bench_path} --sycl --vector-size {str(vec_size)} --reps "
78+
f"{str(reps)} {bench_out}"
79+
)
80+
else:
81+
command = (
82+
f"BENCHMARK_FILTER={bench_filter} I_MPI_PIN_DOMAIN={pin_domain} "
83+
f"I_MPI_PIN_ORDER={pin_order} I_MPI_PIN_CELL={pin_cell} {mpirun} "
84+
f"{n} {bench_path} --vector-size {str(vec_size)} --reps "
85+
f"{str(reps)} {bench_out}"
86+
)
87+
88+
print_run_command(command)
89+
90+
91+
@click.command()
92+
@bench_common_params
93+
def run_shp(
94+
vec_size: int,
95+
reps: int,
96+
d: int,
97+
bench_filter: str = '',
98+
kmp_aff: str = '',
99+
):
100+
bench_path = "./benchmarks/gbench/shp/shp-bench"
101+
time_now_string = datetime.datetime.now().isoformat(timespec='minutes')
102+
directory_path = "benchmarks/json_output/shp"
103+
if not os.path.exists(directory_path):
104+
os.makedirs(directory_path)
105+
bench_out = (
106+
f"--benchmark_out={directory_path}"
107+
f"/shp_benchmark_{time_now_string}.json "
108+
f"--benchmark_out_format=json"
109+
)
110+
command = (
111+
f"BENCHMARK_FILTER={bench_filter} KMP_AFFINITY={kmp_aff} {bench_path} "
112+
f"-d {str(d)} --vector-size {str(vec_size)} --reps {str(reps)} "
113+
f"{bench_out}"
114+
)
115+
print_run_command(command)
116+
117+
118+
def run_all_benchmarks_fsycl_O3():
119+
kmp_affinity = "compact"
120+
121+
# shp
122+
d = 1
123+
run_shp(d=d)
124+
run_shp(d=d, kmp_aff=kmp_affinity)
125+
126+
d = 2
127+
run_shp(d=d)
128+
run_shp(d=d, kmp_aff=kmp_affinity)
129+
130+
# mhp/cpu
131+
sycl_used = False
132+
mpirun = False
133+
run_mhp(sycl_used=sycl_used, mpirun=False)
134+
n = 24
135+
mpi_pin_domain = "core"
136+
mpi_pin_order = "compact"
137+
mpi_pin_cell = "unit"
138+
139+
mpirun = True
140+
run_mhp(
141+
sycl_used=sycl_used,
142+
mpirun=mpirun,
143+
n=n,
144+
pin_domain=mpi_pin_domain,
145+
pin_order=mpi_pin_order,
146+
pin_cell=mpi_pin_cell,
147+
)
148+
n = 48
149+
run_mhp(
150+
sycl_used=sycl_used,
151+
mpirun=mpirun,
152+
n=n,
153+
pin_domain=mpi_pin_domain,
154+
pin_order=mpi_pin_order,
155+
pin_cell=mpi_pin_cell,
156+
)
157+
158+
# mhp/sycl
159+
mpi_pin_domain = "socket"
160+
sycl_used = True
161+
n = 1
162+
run_mhp(
163+
sycl_used=sycl_used,
164+
n=n,
165+
pin_domain=mpi_pin_domain,
166+
mpirun=mpirun,
167+
)
168+
run_mhp(sycl_used=sycl_used, n=n, mpirun=mpirun)
169+
n = 2
170+
run_mhp(
171+
sycl_used=sycl_used,
172+
n=n,
173+
pin_domain=mpi_pin_domain,
174+
pin_order=mpi_pin_order,
175+
pin_cell=mpi_pin_cell,
176+
mpirun=mpirun,
177+
)
178+
179+
180+
def run_all_benchmarks_other_options():
181+
kmp_affinity = "compact"
182+
mpirun = True
183+
# shp
184+
d = 1
185+
run_shp(d=d, kmp_aff=kmp_affinity)
186+
d = 2
187+
run_shp(d=d, kmp_aff=kmp_affinity)
188+
# mhp/cpu
189+
sycl_used = False
190+
mpi_pin_domain = "core"
191+
mpi_pin_order = "compact"
192+
mpi_pin_cell = "unit"
193+
n = 24
194+
run_mhp(
195+
sycl_used=sycl_used,
196+
mpirun=mpirun,
197+
n=n,
198+
pin_domain=mpi_pin_domain,
199+
pin_order=mpi_pin_order,
200+
pin_cell=mpi_pin_cell,
201+
)
202+
n = 48
203+
run_mhp(
204+
sycl_used=sycl_used,
205+
mpirun=mpirun,
206+
n=n,
207+
pin_domain=mpi_pin_domain,
208+
pin_order=mpi_pin_order,
209+
pin_cell=mpi_pin_cell,
210+
)
211+
212+
213+
def run_mhp_test():
214+
sycl_used = False
215+
mpirun = False
216+
n = 1
217+
run_mhp(
218+
sycl_used=sycl_used,
219+
mpirun=mpirun,
220+
n=n,
221+
)
222+
223+
224+
def run_shp_test():
225+
d = 1
226+
run_shp(d=d)
227+
228+
229+
run_mhp_test()
230+
run_shp_test()

‎scripts/devcloud-daily.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#! /bin/bash
2+
3+
# SPDX-FileCopyrightText: Intel Corporation
4+
#
5+
# SPDX-License-Identifier: BSD-3-Clause
6+
source /opt/intel/oneapi/setvars.sh
7+
set -e
8+
hostname
9+
CXX=icpx cmake -DENABLE_SYCL:BOOL=ON -DENABLE_SYCL_MPI:BOOL=ON -B build -S .
10+
make -C build -j
11+
12+
# python -m pip install --upgrade pip
13+
cd build
14+
# pip install -r requirements.txt
15+
python3 ../benchmarks/runner/run_benchmarks.py --vec_size=1000000 --reps=100 --bench_filter='Stream_'

0 commit comments

Comments
 (0)