Add TEA Benchmarks (#2438)

* Add GBench skeleton

* Clean dead code & Fix Compilation issues

* Add Google Benchmark as a submodule

* Use Google Benchmark as a submodule

* Remove GBench global buster installation

* Change results output dir

* Run `benchmarks_tea` during `runall.sh`

* Trigger pipeline run on `tea` changes

Author: PROFeNoM

.gitlab/benchmarks.yml

@@ -58,6 +58,7 @@ benchmarks-tracer:
           - zend_abstract_interface/**/*
           - tests/Benchmarks/**/*
           - benchmark/*
+          - tea/**/*
         compare_to: "master"
       when: on_success
     - when: manual

.gitmodules

@@ -11,3 +11,6 @@
 [submodule "appsec/third_party/libddwaf"]
 	path = appsec/third_party/libddwaf
 	url = https://github.com/DataDog/libddwaf.git
+[submodule "tea/benchmarks/google-benchmark"]
+	path = tea/benchmarks/google-benchmark
+	url = https://github.com/google/benchmark.git

Makefile

@@ -8,7 +8,12 @@ BUILD_DIR = $(PROJECT_ROOT)/tmp/build_$(BUILD_SUFFIX)
 ZAI_BUILD_DIR = $(PROJECT_ROOT)/tmp/build_zai$(if $(ASAN),_asan)
 TEA_BUILD_DIR = $(PROJECT_ROOT)/tmp/build_tea$(if $(ASAN),_asan)
 TEA_INSTALL_DIR = $(TEA_BUILD_DIR)/opt
-TEA_BUILD_TESTS = ON
+TEA_BUILD_TESTS ?= OFF
+TEA_BUILD_BENCHMARKS ?= OFF
+TEA_BENCHMARK_REPETITIONS ?= 10
+# Note: If the tea benchmark format or output is changed, make changes to ./benchmark/runall.sh
+TEA_BENCHMARK_FORMAT ?= json
+TEA_BENCHMARK_OUTPUT ?= $(PROJECT_ROOT)/tea/benchmarks/reports/tea-bench-results.$(TEA_BENCHMARK_FORMAT)
 COMPONENTS_BUILD_DIR = $(PROJECT_ROOT)/tmp/build_components
 SO_FILE = $(BUILD_DIR)/modules/ddtrace.so
 WALL_FLAGS = -Wall -Wextra
@@ -153,16 +158,25 @@ test_extension_ci: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES)
 	$(RUN_TESTS_CMD) -d extension=$(SO_FILE) -m -s $$TEST_PHP_OUTPUT $(BUILD_DIR)/$(TESTS) && ! grep -e 'LEAKED TEST SUMMARY' $$TEST_PHP_OUTPUT; \
 	)
 
-build_tea:
+build_tea: TEA_BUILD_TESTS=ON
+build_tea: TEA_PREFIX_PATH=/opt/catch2
+build_tea: build_tea_common
+
+build_tea_benchmarks: TEA_BUILD_BENCHMARKS=ON
+build_tea_benchmarks: TEA_PREFIX_PATH=/opt/gbench
+build_tea_benchmarks: build_tea_common
+
+build_tea_common:
 	$(Q) test -f $(TEA_BUILD_DIR)/.built || \
 	( \
 		mkdir -p "$(TEA_BUILD_DIR)" "$(TEA_INSTALL_DIR)"; \
 		cd $(TEA_BUILD_DIR); \
-		CMAKE_PREFIX_PATH=/opt/catch2 \
+		CMAKE_PREFIX_PATH=$(TEA_PREFIX_PATH) \
 		cmake \
 			-DCMAKE_INSTALL_PREFIX=$(TEA_INSTALL_DIR) \
 			-DCMAKE_BUILD_TYPE=Debug \
 			-DBUILD_TEA_TESTING=$(TEA_BUILD_TESTS) \
+			-DBUILD_TEA_BENCHMARKING=$(TEA_BUILD_BENCHMARKS) \
 			-DPhpConfig_ROOT=$(shell php-config --prefix) \
 			$(if $(ASAN), -DCMAKE_TOOLCHAIN_FILE=$(PROJECT_ROOT)/cmake/asan.cmake) \
 		$(PROJECT_ROOT)/tea; \
@@ -175,13 +189,21 @@ test_tea: clean_tea build_tea
 		! grep -e "=== Total .* memory leaks detected ===" $(TEA_BUILD_DIR)/Testing/Temporary/LastTest.log; \
 	)
 
+benchmarks_tea: clean_tea build_tea_benchmarks
+	$(TEA_BUILD_DIR)/benchmarks/tea_benchmarks \
+		--benchmark_repetitions=$(TEA_BENCHMARK_REPETITIONS) \
+		--benchmark_out=$(TEA_BENCHMARK_OUTPUT) \
+		--benchmark_format=$(TEA_BENCHMARK_FORMAT) \
+		--benchmark_time_unit=ms
+
 install_tea: build_tea
 	$(Q) test -f $(TEA_BUILD_DIR)/.installed || \
 	( \
 		$(MAKE) -C $(TEA_BUILD_DIR) install; \
 		touch $(TEA_BUILD_DIR)/.installed; \
 	)
 
+build_tea_coverage: TEA_BUILD_TESTS=ON
 build_tea_coverage:
 	$(Q) test -f $(TEA_BUILD_DIR)/.built.coverage || \
 	( \

benchmark/runall.sh

@@ -25,4 +25,8 @@ elif [ "$SCENARIO" = "tracer" ]; then
   ## OPCache Benchmarks
   make benchmarks_opcache
   cp tests/Benchmarks/reports/tracer-bench-results-opcache.csv "$ARTIFACTS_DIR"
+
+  ## Request Startup/Shutdown Benchmarks
+  make benchmarks_tea
+  cp tea/benchmarks/reports/tea-bench-results.json "$ARTIFACTS_DIR"
 fi

tea/CMakeLists.txt

@@ -85,6 +85,11 @@ if(${BUILD_TEA_TESTING})
   add_subdirectory(tests)
 endif()
 
+option(BUILD_TEA_BENCHMARKING "Enable TEA benchmarks" OFF)
+if(${BUILD_TEA_BENCHMARKING})
+  add_subdirectory(benchmarks)
+endif()
+
 # Copy the include files on install
 install(
   FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/common.h
@@ -95,6 +100,9 @@ install(
         ${CMAKE_CURRENT_SOURCE_DIR}/include/sapi.h
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/tea)
 
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/testing/fixture.hpp
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/tea/testing)
+
 install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/testing/catch2.hpp
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/tea/testing)
 

tea/benchmarks/CMakeLists.txt

@@ -0,0 +1,14 @@
+enable_language(CXX)
+
+set(CMAKE_CXX_STANDARD 11)
+
+add_executable(tea_benchmarks benchmark.cc)
+
+# Google Benchmark is included as a git submodule.
+# It depends on Google Test, which it will download if this option is set.
+set(BENCHMARK_DOWNLOAD_DEPENDENCIES ON)
+# Don't build Google Benchmark's unit tests.
+set(BENCHMARK_ENABLE_TESTING OFF)
+add_subdirectory(google-benchmark)
+
+target_link_libraries(tea_benchmarks PUBLIC benchmark::benchmark Tea::Tea)

tea/benchmarks/README.md

@@ -0,0 +1,25 @@
+# Microbenchmarks
+
+This directory contains the request **startup** and **shutdown** microbenchmarks.
+
+The benchmarks uses [Google Benchmark](https://github.com/google/benchmark), whose source is included as a git submodule under `./google-benchmark`.
+
+## How to run benchmarks
+
+First refer to the [CONTRIBUTING.md](../../CONTRIBUTING.md) file to setup the environment.
+
+Then you can run the benchmarks with the following command from the root of the repository:
+
+```bash
+make benchmarks_tea
+```
+
+## How to add a new benchmark
+
+The benchmarks are located in the [benchmark.cc](./benchmark.cc) file and are written using [Google Benchmark](https://github.com/google/benchmark) (v1.8.3).
+
+To add a new benchmark, create a new function in the [benchmark.cc](./benchmark.cc) file. Please, refer to the [User Guide](https://github.com/google/benchmark/blob/main/docs/user_guide.md) for more information.
+
+## Results
+
+The results of the benchmark are stored under the [reports](./reports) folder.

tea/benchmarks/benchmark.cc

@@ -0,0 +1,31 @@
+#include <iostream>
+#include <memory>
+#include <benchmark/benchmark.h>
+#include <include/testing/fixture.hpp>
+#include <Zend/zend_exceptions.h>
+
+static void BM_TeaSapiSpinup(benchmark::State& state) {
+    TeaTestCaseFixture fixture;
+    for (auto _ : state) {
+        fixture.tea_sapi_spinup();
+
+        state.PauseTiming();
+        fixture.tea_sapi_spindown();
+        state.ResumeTiming();
+    }
+}
+BENCHMARK(BM_TeaSapiSpinup);
+
+static void BM_TeaSapiSpindown(benchmark::State& state) {
+    TeaTestCaseFixture fixture;
+    for (auto _ : state) {
+        state.PauseTiming();
+        fixture.tea_sapi_spinup();
+        state.ResumeTiming();
+
+        fixture.tea_sapi_spindown();
+    }
+}
+BENCHMARK(BM_TeaSapiSpindown);
+
+BENCHMARK_MAIN();

tea/benchmarks/google-benchmark

@@ -21,7 +21,7 @@ if test "$PHP_TEA" != "no"; then
     AC_DEFINE(HAVE_TEA, 1, [ ])
 
     PHP_TEA_INCLUDE="sapi/tea/include"
-    PHP_TEA_HEADERS="common.h error.h exceptions.h extension.h frame.h sapi.h testing/catch2.hpp"
+    PHP_TEA_HEADERS="common.h error.h exceptions.h extension.h frame.h sapi.h testing/fixture.hpp testing/catch2.hpp"
     PHP_TEA_CFLAGS="-I${abs_srcdir}/${PHP_TEA_INCLUDE}"
     PHP_TEA_FILES="src/error.c src/exceptions.c src/frame.c src/io.c src/ini.c src/extension.c src/sapi.c"
 

tea/benchmarks/reports/.gitkeep

@@ -38,9 +38,7 @@
 *    Code or cases tested with bailout will fail if code does not bail
 *    Code or cases tested without bailout will fail if code bails
 */
-extern "C" {
-#include "../sapi.h"
-}
+#include "fixture.hpp"
 
 #include <catch2/catch.hpp>
 
@@ -65,92 +63,6 @@ extern "C" {
     __TEA_TEST_CASE_SUITE " [" __TEA_TEST_CASE_DESCRIPTION "]",      \
     "[" __TEA_TEST_CASE_SUITE "]" __TEA_TEST_CASE_TAGS /* }}} */
 
-/* {{{ Test Case Fixing */
-typedef enum {
-    TEA_TEST_CASE_STAGE_INITIAL =  0b000,
-    TEA_TEST_CASE_STAGE_PROLOGUE = 0b001,
-    TEA_TEST_CASE_STAGE_PREFORK =  0b010,
-    TEA_TEST_CASE_STAGE_REQUEST =  0b100
-} TeaTestCaseStage;
-
-class TeaTestCaseFixture {
-public:
-    TeaTestCaseFixture() {
-       stage = TEA_TEST_CASE_STAGE_INITIAL;
-    }
-
-    bool tea_sapi_sinit() {
-        if (::tea_sapi_sinit()) {
-            stage |= TEA_TEST_CASE_STAGE_PROLOGUE;
-            return true;
-        }
-        return false;
-    }
-
-    bool tea_sapi_minit() {
-        if (::tea_sapi_minit()) {
-            stage |= TEA_TEST_CASE_STAGE_PREFORK;
-            return true;
-        }
-        return false;
-    }
-
-    bool tea_sapi_rinit() {
-        if (::tea_sapi_rinit()) {
-            stage |= TEA_TEST_CASE_STAGE_REQUEST;
-            return true;
-        }
-        return false;
-    }
-
-    bool tea_sapi_spinup() {
-        return tea_sapi_sinit() &&
-               tea_sapi_minit() &&
-               tea_sapi_rinit();
-    }
-
-    void tea_sapi_rshutdown() {
-        stage &= ~TEA_TEST_CASE_STAGE_REQUEST;
-
-        ::tea_sapi_rshutdown();
-    }
-
-    void tea_sapi_mshutdown() {
-        stage &= ~TEA_TEST_CASE_STAGE_PREFORK;
-
-        ::tea_sapi_mshutdown();
-    }
-
-    void tea_sapi_sshutdown() {
-        stage &= ~TEA_TEST_CASE_STAGE_PROLOGUE;
-
-        ::tea_sapi_sshutdown();
-    }
-
-    void tea_sapi_spindown() {
-        tea_sapi_rshutdown();
-        tea_sapi_mshutdown();
-        tea_sapi_sshutdown();
-    }
-
-    virtual ~TeaTestCaseFixture() {
-        if (stage & TEA_TEST_CASE_STAGE_REQUEST) {
-            tea_sapi_rshutdown();
-        }
-
-        if (stage & TEA_TEST_CASE_STAGE_PREFORK) {
-            tea_sapi_mshutdown();
-        }
-
-        if (stage & TEA_TEST_CASE_STAGE_PROLOGUE) {
-            tea_sapi_sshutdown();
-        }
-    }
-
-private:
-    unsigned int stage;
-};
-
 #define TEA_TEST_CASE_DECL(...) \
     TEST_CASE_METHOD(           \
         TeaTestCaseFixture,     \

tea/config.m4

@@ -0,0 +1,94 @@
+#ifndef HAVE_TEA_FIXTURE
+#define HAVE_TEA_FIXTURE
+
+extern "C" {
+#include "../sapi.h"
+};
+
+/* {{{ Test Case Fixing */
+typedef enum {
+    TEA_TEST_CASE_STAGE_INITIAL =  0b000,
+    TEA_TEST_CASE_STAGE_PROLOGUE = 0b001,
+    TEA_TEST_CASE_STAGE_PREFORK =  0b010,
+    TEA_TEST_CASE_STAGE_REQUEST =  0b100
+} TeaTestCaseStage;
+
+class TeaTestCaseFixture {
+public:
+    TeaTestCaseFixture() {
+        stage = TEA_TEST_CASE_STAGE_INITIAL;
+    }
+
+    bool tea_sapi_sinit() {
+        if (::tea_sapi_sinit()) {
+            stage |= TEA_TEST_CASE_STAGE_PROLOGUE;
+            return true;
+        }
+        return false;
+    }
+
+    bool tea_sapi_minit() {
+        if (::tea_sapi_minit()) {
+            stage |= TEA_TEST_CASE_STAGE_PREFORK;
+            return true;
+        }
+        return false;
+    }
+
+    bool tea_sapi_rinit() {
+        if (::tea_sapi_rinit()) {
+            stage |= TEA_TEST_CASE_STAGE_REQUEST;
+            return true;
+        }
+        return false;
+    }
+
+    bool tea_sapi_spinup() {
+        return tea_sapi_sinit() &&
+               tea_sapi_minit() &&
+               tea_sapi_rinit();
+    }
+
+    void tea_sapi_rshutdown() {
+        stage &= ~TEA_TEST_CASE_STAGE_REQUEST;
+
+        ::tea_sapi_rshutdown();
+    }
+
+    void tea_sapi_mshutdown() {
+        stage &= ~TEA_TEST_CASE_STAGE_PREFORK;
+
+        ::tea_sapi_mshutdown();
+    }
+
+    void tea_sapi_sshutdown() {
+        stage &= ~TEA_TEST_CASE_STAGE_PROLOGUE;
+
+        ::tea_sapi_sshutdown();
+    }
+
+    void tea_sapi_spindown() {
+        tea_sapi_rshutdown();
+        tea_sapi_mshutdown();
+        tea_sapi_sshutdown();
+    }
+
+    virtual ~TeaTestCaseFixture() {
+        if (stage & TEA_TEST_CASE_STAGE_REQUEST) {
+            tea_sapi_rshutdown();
+        }
+
+        if (stage & TEA_TEST_CASE_STAGE_PREFORK) {
+            tea_sapi_mshutdown();
+        }
+
+        if (stage & TEA_TEST_CASE_STAGE_PROLOGUE) {
+            tea_sapi_sshutdown();
+        }
+    }
+
+private:
+    unsigned int stage;
+};
+
+#endif  // HAVE_TEA_FIXTURE