Skip to content

Commit 67a43df

Browse files
kwkkwk
authored
Add mechanism to compare compile-time perf (#672)
This change adds a directory `script/perf` that contains files to run a compile-time performance test for snapshots (called "big-merge"), PGOed snapshots setup and the system LLVM. To run the performance test for the current day, run the following steps: ```console $ cd scripts/perf $ make ``` You'll be prompted with a markdown output that you can copy and paste into a github issue. The output is located between `<!--BEGIN REPORT-->` and `<!--END REPORT-->` markers.
1 parent e71f830 commit 67a43df

File tree

4 files changed

+264
-0
lines changed

4 files changed

+264
-0
lines changed

‎scripts/perf/Containerfile

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
FROM fedora:rawhide
2+
LABEL description="Test compilers with llvm-test-suite"
3+
4+
USER root
5+
WORKDIR /root
6+
7+
# Install deps to run test-suite
8+
RUN dnf install -y 'dnf-command(copr)' \
9+
&& dnf copr enable -y @fedora-llvm-team/llvm-test-suite \
10+
&& dnf install -y \
11+
clang \
12+
cmake \
13+
coreutils \
14+
envsubst \
15+
fedora-packager \
16+
git \
17+
jq \
18+
jq \
19+
llvm-test-suite \
20+
ninja-build \
21+
python3-lit \
22+
python3-pandas \
23+
python3-pip \
24+
python3-scipy \
25+
python3-setuptools \
26+
python3-virtualenv \
27+
tcl \
28+
tcl-devel \
29+
tcl-tclreadline \
30+
tcl-thread-devel \
31+
tcl-zlib \
32+
which
33+
34+
COPY entrypoint.sh /root/entrypoint.sh
35+
USER root
36+
ENTRYPOINT [ "/root/entrypoint.sh" ]

‎scripts/perf/Makefile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
.PHONY: all
2+
all: setup run
3+
4+
.PHONY: setup
5+
setup:
6+
podman build -t evaluation .
7+
8+
.PHONY: run
9+
run:
10+
podman run -it --rm evaluation

‎scripts/perf/README.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# README
2+
3+
This container setup allows you to compare compile time performance of the
4+
system llvm against "big-merge" (aka snapshot) and "pgo" for the current date.
5+
6+
# How to
7+
8+
Just run `make` to build and run the container image. It takes a long time to complete.
9+
10+
Then you'll be prompted with a markdown output that you can copy and paste into a github issue.
11+
12+
The output is located between `<!--BEGIN REPORT-->` and `<!--END REPORT-->` markers.

‎scripts/perf/entrypoint.sh

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
#!/usr/bin/bash
2+
3+
set -x
4+
set -e
5+
6+
function configure_build_run {
7+
# See also https://llvm.org/docs/TestSuiteGuide.html#common-configuration-options
8+
cmake_args=""
9+
10+
# Recommended setting for compile-time benchmarks
11+
cmake_args="$cmake_args -DTEST_SUITE_SUBDIRS=CTMark"
12+
13+
# For PGO performance comparison we expect differences in the range of 10%
14+
# and more. Therefore we don't need perf.
15+
cmake_args="$cmake_args -DTEST_SUITE_USE_PERF=OFF"
16+
17+
# We want to measure the run-time of the compiler and therefore don't have
18+
# to "run" the benchmarks. We just need to compile them.
19+
cmake_args="$cmake_args -DTEST_SUITE_RUN_BENCHMARKS=OFF"
20+
21+
# Collect internal LLVM statistics. Appends -save-stats=obj when invoking
22+
# the compiler and makes the lit runner collect and merge the statistic
23+
# files.
24+
cmake_args="$cmake_args -DTEST_SUITE_COLLECT_STATS=ON"
25+
26+
# Some programs are unsuitable for performance measurements. Setting the
27+
# TEST_SUITE_BENCHMARKING_ONLY CMake option to ON will disable them.
28+
cmake_args="$cmake_args -DTEST_SUITE_BENCHMARKING_ONLY=ON"
29+
30+
# Configure the test suite
31+
cmake \
32+
-GNinja \
33+
-DCMAKE_C_COMPILER=/usr/bin/clang \
34+
-DCMAKE_CXX_COMPILER=/usr/bin/clang++ \
35+
$cmake_args \
36+
-C/usr/share/llvm-test-suite/cmake/caches/O3.cmake \
37+
/usr/share/llvm-test-suite
38+
39+
# Build the test-suite with one job at a time for a fair comparison.
40+
ninja -j1
41+
42+
# Run the tests with lit:
43+
lit -v -o results.json . || true
44+
}
45+
46+
function get_llvm_version() {
47+
clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9](pre[0-9]{8}.g[0-9a-f]{14})?' | head -n1
48+
}
49+
50+
# Query version information for given day
51+
yyyymmdd=$(date +%Y%m%d)
52+
git_rev=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-git-revision-${yyyymmdd}.txt)
53+
git_rev_short="${git_rev:0:14}"
54+
llvm_release=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-release-${yyyymmdd}.txt)
55+
rpm_suffix="${llvm_release}~pre${yyyymmdd}.g${git_rev_short}"
56+
57+
echo "git_rev=$git_rev"
58+
echo "git_rev_short=$git_rev_short"
59+
echo "llvm_release=$llvm_release"
60+
echo "rpm_suffix=$rpm_suffix"
61+
62+
######################################################################################
63+
# PGO
64+
######################################################################################
65+
66+
# Install and enable the repository that provides the PGO LLVM Toolchain
67+
# See https://llvm.org/docs/HowToBuildWithPGO.html#building-clang-with-pgo
68+
dnf copr enable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd}
69+
repo_file=$(dnf repoinfo --json *llvm-snapshots-pgo* | jq -r ".[0].repo_file_path")
70+
distname=$(rpm --eval "%{?fedora:fedora}%{?rhel:rhel}") envsubst '$distname' < $repo_file > /tmp/new_repo_file
71+
cat /tmp/new_repo_file > $repo_file
72+
dnf -y install \
73+
clang-${rpm_suffix} \
74+
clang-${rpm_suffix} \
75+
clang-libs-${rpm_suffix} \
76+
clang-resource-filesystem-${rpm_suffix} \
77+
llvm-${rpm_suffix} \
78+
llvm-libs-${rpm_suffix} \
79+
llvm-test-suite
80+
81+
pgo_version=$(get_llvm_version)
82+
83+
mkdir -pv ~/pgo
84+
cd ~/pgo
85+
86+
configure_build_run
87+
88+
# Remove packages from that PGO repo and the repo itself
89+
repo_pkgs_installed=$(dnf repoquery --installed --queryformat ' %{name} %{from_repo} ' | grep -Po "[^ ]+ [^ ]+llvm-snapshots-pgo" | awk '{print $1}')
90+
dnf -y remove $repo_pkgs_installed;
91+
dnf copr disable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd}
92+
93+
######################################################################################
94+
# big-merge
95+
######################################################################################
96+
97+
# Install and enable the repository that provides the big-merge LLVM Toolchain
98+
dnf copr enable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd}
99+
repo_file=$(dnf repoinfo --json *llvm-snapshots-big-merge* | jq -r ".[0].repo_file_path")
100+
distname=$(rpm --eval "%{?fedora:fedora}%{?rhel:rhel}") envsubst '$distname' < $repo_file > /tmp/new_repo_file
101+
cat /tmp/new_repo_file > $repo_file
102+
dnf -y install \
103+
clang-${rpm_suffix} \
104+
clang-${rpm_suffix} \
105+
clang-libs-${rpm_suffix} \
106+
clang-resource-filesystem-${rpm_suffix} \
107+
llvm-${rpm_suffix} \
108+
llvm-libs-${rpm_suffix} \
109+
llvm-test-suite
110+
111+
big_merge_version=$(get_llvm_version)
112+
113+
mkdir -pv ~/big-merge
114+
cd ~/big-merge
115+
116+
configure_build_run
117+
# Remove packages from that big-merge repo and the repo itself
118+
repo_pkgs_installed=$(dnf repoquery --installed --queryformat ' %{name} %{from_repo} ' | grep -Po "[^ ]+ [^ ]+llvm-snapshots-big-merge" | awk '{print $1}')
119+
dnf -y remove $repo_pkgs_installed;
120+
dnf copr disable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd}
121+
122+
######################################################################################
123+
# system llvm
124+
######################################################################################
125+
126+
# Build with regular clang
127+
dnf install -y clang clang-libs clang-resource-filesystem llvm llvm-libs llvm-test-suite
128+
system_version=$(get_llvm_version)
129+
mkdir -pv ~/system
130+
cd ~/system
131+
132+
configure_build_run
133+
134+
/usr/share/llvm-test-suite/utils/compare.py \
135+
--metric compile_time \
136+
--lhs-name ${system_version} \
137+
--rhs-name pgo-${yyyymmdd} \
138+
~/system/results.json vs ~/pgo/results.json > ~/results-system-vs-pgo.txt || true
139+
140+
/usr/share/llvm-test-suite/utils/compare.py \
141+
--metric compile_time \
142+
--lhs-name ${system_version} \
143+
--rhs-name big-merge-${yyyymmdd} \
144+
~/system/results.json vs ~/big-merge/results.json > ~/results-system-vs-big-merge.txt || true
145+
146+
/usr/share/llvm-test-suite/utils/compare.py \
147+
--metric compile_time \
148+
--lhs-name big-merge \
149+
--rhs-name pgo-${yyyymmdd} \
150+
~/big-merge/results.json vs ~/pgo/results.json > ~/results-big-merge-vs-pgo.txt || true
151+
152+
153+
set +x
154+
155+
function print_report() {
156+
# calculate min/max for y-axis in diagram with some padding
157+
a=$(grep -ioP "Geomean difference\s+\K(-)?[0-9]+\.[0-9]+" ~/results-big-merge-vs-pgo.txt)
158+
b=$(grep -ioP "Geomean difference\s+\K(-)?[0-9]+\.[0-9]+" ~/results-system-vs-big-merge.txt)
159+
c=$(grep -ioP "Geomean difference\s+\K(-)?[0-9]+\.[0-9]+" ~/results-system-vs-pgo.txt)
160+
a=$(python3 -c "print(-1*$a)")
161+
b=$(python3 -c "print(-1*$b)")
162+
c=$(python3 -c "print(-1*$c)")
163+
pad=5
164+
min=$(python3 -c "print(min($a,$b,$c)-$pad)")
165+
max=$(python3 -c "print(max($a,$b,$c)+$pad)")
166+
167+
redhat_release=$(cat /etc/redhat-release)
168+
arch=$(uname -m)
169+
170+
echo '<!--BEGIN REPORT-->'
171+
cat <<EOF
172+
\`\`\`mermaid
173+
xychart-beta horizontal
174+
title "Compile time performance (${yyyymmdd}, ${arch}, ${redhat_release})"
175+
x-axis ["PGO vs. big-merge", "PGO vs. system", "big-merge vs. system"]
176+
y-axis "Geomean performance (in %)" ${min} --> ${max}
177+
bar [${a}, ${c}, ${b}]
178+
line [${a}, ${c}, ${b}]
179+
\`\`\`
180+
181+
<details>
182+
<summary>Compile time results for ${yyyymmdd}</summary>
183+
184+
<h2>big-merge (${big_merge_version}) vs. PGO (${pgo_version})</h2>
185+
186+
\`\`\`
187+
$(cat ~/results-big-merge-vs-pgo.txt)
188+
\`\`\`
189+
190+
<h2>System (${system_version}) vs. big-merge (${big_merge_version})</h2>
191+
192+
\`\`\`
193+
$(cat ~/results-system-vs-big-merge.txt)
194+
\`\`\`
195+
196+
<h2>System (${system_version}) vs. PGO (${pgo_version})</h2>
197+
198+
\`\`\`
199+
$(cat ~/results-system-vs-pgo.txt)
200+
\`\`\`
201+
</details>
202+
EOF
203+
echo '<!--END REPORT-->'
204+
}
205+
206+
print_report

0 commit comments

Comments
 (0)