Skip to content

Commit 3e2aeea

Browse files
author
joeylee.lz
committed
Add FrameworkBenchmarks performance test
1 parent c671a7f commit 3e2aeea

File tree

6 files changed

+199
-22
lines changed

6 files changed

+199
-22
lines changed

‎perf/README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ subdirectory and given a meaningful name. Once the reorganization of this direc
2626
//├── dacapo
2727
//├── renaissance
2828
//├── liberty
29+
//├── FrameworkBenchmarks
2930
3031
```
3132
Each subdirectory requires a build.xml file describing where to pull the benchmark suite from, and how to build and run it. Each subdirectory also requires a playlist.xml file which describes 1 or more benchmarks and what commands to run in order to execute a particular benchmark run.
@@ -49,4 +50,7 @@ Liberty benchmarks from https://github.com/OpenLiberty - including liberty-dt7-s
4950
#### renaissance
5051
Renaissance benchmarks from https://github.com/renaissance-benchmarks/renaissance - including renaissance-akka-uct, renaissance-als, renaissance-chi-square, renaissance-db-shootout, renaissance-dec-tree, renaissance-finagle-chirper, renaissance-finagle-http, renaissance-fj-kmeans, renaissance-future-genetic, renaissance-gauss-mix, renaissance-log-regression, renaissance-mnemonics, renaissance-movie-lens, renaissance-naive-bayes, renaissance-par-mnemonics, renaissance-philosophers and renaissance-scala-kmeans
5152

53+
#### FrameworkBenchmarks
54+
FrameworkBenchmarks from https://github.com/TechEmpower/FrameworkBenchmarks - including a wide field of web application frameworks for JAVA.
55+
5256
Additional benchmarks are being reviewed for addition and if you wish to include more, please comment in the open performance benchmarks [issue 1112](https://github.com/adoptium/aqa-tests/issues/1112).

‎perf/framework-benchmarks/README.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# FrameworkBenchmarks
2+
3+
[FrameworkBenchmarks](https://github.com/TechEmpower/FrameworkBenchmarks/tree/master/frameworks/Java) provides representative performance measures across a wide field of web application frameworks. With much help from the community, coverage is quite broad and we are happy to broaden it further with contributions.
4+
5+
Here we focus on the wide range of Java frameworks for web applications scenarios.
6+
7+
FrameworkBenchmarks covers common workloads as plaintext processing, json content parsing, popular database manipulations
8+
for different frameworks.
9+
10+
Notice the different from external tests:
11+
```aidl
12+
external tests runs framework inner unit tests to ensure frameworks behaving correctly.
13+
perf framework tests stresses common worloads to ensure frameworks are performing well under test VMs.
14+
```
15+
16+
We are enabling the following list of frameworks to avoid regression:
17+
18+
| Framework | Workloads |
19+
|-----------|----------------------------------------|
20+
| netty | plaintext,json |
21+
| quarkus | plaintext,json,query,fortune,update,db |
22+
| spark | plaintext,json,query,fortune,update,db |
23+
| spring | plaintext,json,query,fortune,update,db |
24+
| spring-jpa | plaintext,json,query,fortune,update,db |
25+
| servlet | plaintext,json |
26+
| servlet3 | plaintext,json |

‎perf/framework-benchmarks/build.xml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<?xml version="1.0"?>
2+
<project name="FrameworkBenchmarks-Test" default="build" basedir=".">
3+
<taskdef resource="net/sf/antcontrib/antlib.xml" />
4+
<description>
5+
Build FrameworkBenchmarks-Test Docker image
6+
</description>
7+
8+
<!-- set properties for this build -->
9+
<property name="TEST" value="framework-benchmarks" />
10+
<property name="DEST" value="${BUILD_ROOT}/perf/${TEST}" />
11+
<property name="src" location="." />
12+
13+
<condition property="git-prefix" value="git" else="https">
14+
<isset property="isZOS"/>
15+
</condition>
16+
17+
<target name="init">
18+
<mkdir dir="${DEST}"/>
19+
</target>
20+
21+
<target name="getFrameworkBenchmarks" depends="init" description="Clone the distribution">
22+
<echo message="Cloning FrameworkBenchmarks"/>
23+
<var name="git_command" value="clone --depth 1 -b master ${git-prefix}://github.com/TechEmpower/FrameworkBenchmarks.git ${DEST}"/>
24+
<echo message="git ${git_command}" />
25+
<exec executable="git" failonerror="false" dir=".">
26+
<arg line="${git_command}" />
27+
</exec>
28+
</target>
29+
30+
<target name="dist" depends="getFrameworkBenchmarks" description="generate the distribution">
31+
<copy todir="${DEST}">
32+
<fileset dir="${src}" includes="*.xml, *.mk"/>
33+
</copy>
34+
</target>
35+
36+
<target name="build">
37+
<antcall target="dist" inheritall="true" />
38+
</target>
39+
</project>

‎perf/framework-benchmarks/playlist.xml

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
<?xml version='1.0' encoding='UTF-8'?>
2+
<!--
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
-->
15+
<playlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../TKG/playlist.xsd">
16+
<test>
17+
<testCaseName>FrameworkBenchmarks-netty</testCaseName>
18+
<command>
19+
${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh netty netty.dockerfile ${TEST_RESROOT}; \
20+
$(TEST_STATUS)
21+
</command>
22+
<levels>
23+
<level>special</level>
24+
</levels>
25+
<groups>
26+
<group>perf</group>
27+
</groups>
28+
</test>
29+
<test>
30+
<testCaseName>FrameworkBenchmarks-quarkus</testCaseName>
31+
<command>
32+
${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh quarkus quarkus.dockerfile ${TEST_RESROOT}; \
33+
$(TEST_STATUS)
34+
</command>
35+
<levels>
36+
<level>special</level>
37+
</levels>
38+
<groups>
39+
<group>perf</group>
40+
</groups>
41+
</test>
42+
<test>
43+
<testCaseName>FrameworkBenchmarks-spring</testCaseName>
44+
<command>
45+
${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spring spring.dockerfile ${TEST_RESROOT}; \
46+
$(TEST_STATUS)
47+
</command>
48+
<levels>
49+
<level>special</level>
50+
</levels>
51+
<groups>
52+
<group>perf</group>
53+
</groups>
54+
</test>
55+
<test>
56+
<testCaseName>FrameworkBenchmarks-spring-jpa</testCaseName>
57+
<command>
58+
${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spring spring-jpa.dockerfile ${TEST_RESROOT}; \
59+
$(TEST_STATUS)
60+
</command>
61+
<levels>
62+
<level>special</level>
63+
</levels>
64+
<groups>
65+
<group>perf</group>
66+
</groups>
67+
</test>
68+
<test>
69+
<testCaseName>FrameworkBenchmarks-spark</testCaseName>
70+
<command>
71+
${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spark spark.dockerfile ${TEST_RESROOT}; \
72+
$(TEST_STATUS)
73+
</command>
74+
<levels>
75+
<level>special</level>
76+
</levels>
77+
<groups>
78+
<group>perf</group>
79+
</groups>
80+
</test>
81+
<test>
82+
<testCaseName>FrameworkBenchmarks-servlet3</testCaseName>
83+
<command>
84+
${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh servlet3 servlet3.dockerfile ${TEST_RESROOT}; \
85+
$(TEST_STATUS)
86+
</command>
87+
<levels>
88+
<level>special</level>
89+
</levels>
90+
<groups>
91+
<group>perf</group>
92+
</groups>
93+
</test>
94+
<test>
95+
<testCaseName>FrameworkBenchmarks-servlet</testCaseName>
96+
<command>
97+
${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh servlet servlet.dockerfile ${TEST_RESROOT}; \
98+
$(TEST_STATUS)
99+
</command>
100+
<levels>
101+
<level>special</level>
102+
</levels>
103+
<groups>
104+
<group>perf</group>
105+
</groups>
106+
</test>
107+
</playlist>

‎perf/tfb-runner.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
test=$1
2+
dockerfile=$2
3+
TEST_RESROOT=$3
4+
5+
cd ${TEST_RESROOT}
6+
testcase=` echo ${dockerfile} | cut -d'.' -f 1`
7+
dockerfile=${TEST_RESROOT}/frameworks/Java/${test}/${dockerfile}
8+
9+
if [ ! -f ${dockerfile} ]; then
10+
echo "Warning: ${dockerfile} not found"
11+
exit 1
12+
fi
13+
14+
cp -r ${TEST_JDK_HOME} ${TEST_RESROOT}/frameworks/Java/${test}/jdk
15+
awk '/CMD.*/ && !x {print "COPY jdk /opt/java/openjdk"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile}
16+
awk '/CMD.*/ && !x {print "ENV JAVA_HOME=/opt/java/openjdk"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile}
17+
awk '/CMD.*/ && !x {print "ENV PATH=/opt/java/openjdk/bin:$PATH"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile}
18+
cat ${dockerfile}
19+
./tfb --test $testcase
20+
21+
# get removable results
22+
docker run --entrypoint /bin/bash --rm --network tfb -v /var/run/docker.sock:/var/run/docker.sock -v ${TEST_RESROOT}:/FrameworkBenchmarks techempower/tfb -c "chmod -R 777 /FrameworkBenchmarks/results"

‎testenv/testenv.properties

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,2 @@
1-
TKG_REPO=https://github.com/adoptium/TKG.git
2-
TKG_BRANCH=master
31
OPENJ9_REPO=https://github.com/eclipse-openj9/openj9.git
4-
OPENJ9_BRANCH=master
5-
STF_REPO=https://github.com/adoptium/STF.git
6-
STF_BRANCH=master
7-
OPENJ9_SYSTEMTEST_REPO=https://github.com/eclipse-openj9/openj9-systemtest.git
8-
OPENJ9_SYSTEMTEST_BRANCH=master
9-
ADOPTOPENJDK_SYSTEMTEST_REPO=https://github.com/adoptium/aqa-systemtest.git
10-
ADOPTOPENJDK_SYSTEMTEST_BRANCH=master
11-
JDK8_REPO=https://github.com/adoptium/jdk8u.git
12-
JDK8_BRANCH=dev
13-
JDK11_REPO=https://github.com/adoptium/jdk11u.git
14-
JDK11_BRANCH=dev
15-
JDK17_REPO=https://github.com/adoptium/jdk17u.git
16-
JDK17_BRANCH=dev
17-
JDK8_OPENJ9_REPO=https://github.com/ibmruntimes/openj9-openjdk-jdk8.git
18-
JDK8_OPENJ9_BRANCH=openj9
19-
JDK11_OPENJ9_REPO=https://github.com/ibmruntimes/openj9-openjdk-jdk11.git
20-
JDK11_OPENJ9_BRANCH=openj9
21-
JDK17_OPENJ9_REPO=https://github.com/ibmruntimes/openj9-openjdk-jdk17.git
22-
JDK17_OPENJ9_BRANCH=openj9
23-
AQA_REQUIRED_TARGETS=sanity.functional,extended.functional,special.functional,sanity.openjdk,extended.openjdk,sanity.system,extended.system,sanity.perf,extended.perf
2+
OPENJ9_SHA=b8b68fd9224ddb378cdfa2e1fb6cdd86aac62eb1

0 commit comments

Comments
 (0)