From 37efa3b79241d17cf94721a7bb6985525b2db2fd Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Wed, 13 Nov 2024 19:34:13 +0100 Subject: [PATCH 1/2] Create many tiny artifacts for faster download And not escaping the constraints many systems have on /tmp size. If someone installs multiple php versions at once, we still download the full package. The full package also remains available for offline installations of multiple targets for now. Signed-off-by: Bob Weinand --- .circleci/continue_config.yml | 10 +-- Makefile | 2 +- datadog-setup.php | 100 ++++++++++++++++++++----- tooling/bin/generate-final-artifact.sh | 75 ++++++++++++++++--- 4 files changed, 152 insertions(+), 35 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 03107b6d4d..4195bc1dbf 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -3620,8 +3620,8 @@ jobs: name: Setup docker container shell: powershell.exe command: | - mkdir extensions_windows_x86_64 - mkdir extensions_windows_x86_64_debugsymbols + mkdir extensions_x86_64 + mkdir extensions_x86_64_debugsymbols # No DNS for you by default in circleci docker containers docker network create -d "nat" -o com.docker.network.windowsshim.dnsservers="1.1.1.1" net docker run -v ${pwd}:C:\Users\ContainerAdministrator\app -d --network net --name php << parameters.docker_image >> ping -t localhost @@ -3629,7 +3629,7 @@ jobs: name: Build nts shell: powershell.exe command: | - docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb" + docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb" - run: name: Reuse libdatadog build shell: powershell.exe @@ -3639,10 +3639,10 @@ jobs: name: Build zts shell: powershell.exe command: | - docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb" + docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb" - persist_to_workspace: root: '.' - paths: [ './extensions_windows_x86_64', './extensions_windows_x86_64_debugsymbols' ] + paths: [ './extensions_x86_64', './extensions_x86_64_debugsymbols' ] compile_appsec_extension_centos: working_directory: ~/datadog diff --git a/Makefile b/Makefile index ab10335e0f..2b3f5ed405 100644 --- a/Makefile +++ b/Makefile @@ -491,7 +491,7 @@ build_pecl_package: tooling/bin/pecl-build $${FILES//$${BUILD_DIR}/} dbgsym.tar.gz: - $(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_windows_x86_64_debugsymbols --owner=0 --group=0) + $(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_x86_64_debugsymbols --owner=0 --group=0) installer_packages: .apk.x86_64 .apk.aarch64 .rpm.x86_64 .rpm.aarch64 .deb.x86_64 .deb.arm64 .tar.gz.x86_64 .tar.gz.aarch64 bundle.tar.gz dbgsym.tar.gz tar --use-compress-program=pigz --exclude='dd-library-php-ssi-*' -cf packages.tar.gz $(PACKAGES_BUILD_DIR) --owner=0 --group=0 diff --git a/datadog-setup.php b/datadog-setup.php index 051aef8ca1..ac843d617c 100644 --- a/datadog-setup.php +++ b/datadog-setup.php @@ -95,10 +95,11 @@ function print_help() option can be provided multiple times. --install-dir Install to a specific directory. Default: '$installdir' --uninstall Uninstall the library from the specified binaries. + --file Path to a dd-library-php-*.tar.gz file. Can be used for offline installation. --extension-dir Specify the extension directory. Default: PHP's extension directory. --ini Specify the INI file to use. Default: /98-ddtrace.ini --enable-appsec Enable the application security monitoring module. - --enable-profiling Enable the profiling module. + --enable-profiling Enable the profiling module. -d setting[=value] Used in conjunction with `config ` command to specify the INI setting to get or set. @@ -462,9 +463,58 @@ function install($options) $selectedBinaries = require_binaries_or_exit($options); $interactive = empty($options[OPT_PHP_BIN]); + $commandExtensionSuffixes = []; + $downloadVersions = []; + foreach ($selectedBinaries as $command => $fullPath) { + $binaryForLog = ($command === $fullPath) ? $fullPath : "$command ($fullPath)"; + echo "Checking for binary: $binaryForLog\n"; + + check_php_ext_prerequisite_or_exit($fullPath, 'json'); + + $phpProperties = ini_values($fullPath); + if (!isset($phpProperties[INI_SCANDIR])) { + if (!isset($phpProperties[INI_MAIN])) { + if (IS_WINDOWS) { + $phpProperties[INI_MAIN] = dirname($fullPath) . "/php.ini"; + } else { + print_error_and_exit( + "It is not possible to perform installation on this " + . "system because there is no scan directory and no " + . "configuration file loaded." + ); + } + } + + print_warning( + "Performing an installation without a scan directory may " + . "result in fragile installations that are broken by normal " + . "system upgrades. It is advisable to use the configure " + . "switch --with-config-file-scan-dir when building PHP." + ); + } + + // Suffix (zts/debug) + $extensionSuffix = ''; + if (is_truthy($phpProperties[IS_DEBUG])) { + $extensionSuffix .= '-debug'; + } + if (is_truthy($phpProperties[THREAD_SAFETY])) { + $extensionSuffix .= '-zts'; + } + + $commandExtensionSuffixes[$command] = $extensionSuffix; + + $extensionVersion = $phpProperties[PHP_API]; + $downloadVersions["$extensionVersion$extensionSuffix"] = true; + } + + $tar_gz_suffix = ""; + if (count($downloadVersions) === 1) { + $tar_gz_suffix = "-" . key($downloadVersions); + } + // Preparing clean tmp folder to extract files $tmpDir = sys_get_temp_dir() . '/dd-install'; - $tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}.tar.gz"; $tmpArchiveRoot = $tmpDir . '/dd-library-php'; $tmpArchiveTraceRoot = $tmpDir . '/dd-library-php/trace'; $tmpArchiveAppsecRoot = $tmpDir . '/dd-library-php/appsec'; @@ -490,8 +540,14 @@ function install($options) print_warning('--' . OPT_FILE . ' option is intended for internal usage and can be removed without notice'); $tmpDirTarGz = $options[OPT_FILE]; } else { - $url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}.tar.gz"; - download($url, $tmpDirTarGz); + for (;;) { + $url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}{$tar_gz_suffix}.tar.gz"; + $tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}{$tar_gz_suffix}.tar.gz"; + if (download($url, $tmpDirTarGz, $tar_gz_suffix != "")) { + break; + } + $tar_gz_suffix = ""; // retry with the full archive if the original download failed + } } if (!IS_WINDOWS || `where tar 2> nul` !== null) { execute_or_exit( @@ -574,15 +630,7 @@ function install($options) // Copying the extension $extensionVersion = $phpProperties[PHP_API]; - - // Suffix (zts/debug/alpine) - $extensionSuffix = ''; - if (is_truthy($phpProperties[IS_DEBUG])) { - $extensionSuffix .= '-debug'; - } - if (is_truthy($phpProperties[THREAD_SAFETY])) { - $extensionSuffix .= '-zts'; - } + $extensionSuffix = $commandExtensionSuffixes[$command]; $extDir = isset($options[OPT_EXTENSION_DIR]) ? $options[OPT_EXTENSION_DIR] : $phpProperties[EXTENSION_DIR]; echo "Installing extension to $extDir\n"; @@ -1416,7 +1464,7 @@ function execute_or_exit($exitMessage, $command) * @param string $url * @param string $destination */ -function download($url, $destination) +function download($url, $destination, $retry = false) { echo "Downloading installable archive from $url.\n"; echo "This operation might take a while.\n"; @@ -1444,12 +1492,18 @@ function download($url, $destination) curl_setopt($ch, CURLOPT_NOPROGRESS, false); $progress_counter = 0; $return = curl_exec($ch); + + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($httpCode == 404) { + return false; + } + curl_close($ch); fclose($fp); if (false !== $return) { echo $okMessage; - return; + return true; } // Otherwise we attempt other methods } @@ -1461,20 +1515,20 @@ function download($url, $destination) if (!IS_WINDOWS && false !== exec('curl --version', $output, $statusCode) && $statusCode === 0) { $curlInvocationStatusCode = 0; system( - 'curl -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url), + 'curl -f -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url), $curlInvocationStatusCode ); if ($curlInvocationStatusCode === 0) { echo $okMessage; - return; + return true; } // Otherwise we attempt other methods } // file_get_contents if (is_truthy(ini_get('allow_url_fopen')) && extension_loaded('openssl')) { - ini_set("memory_limit", "1G"); // increase memory limit otherwise we may run OOM here. + ini_set("memory_limit", "2G"); // increase memory limit otherwise we may run OOM here. $data = @file_get_contents($url); // PHP doesn't like too long location headers, and on PHP 7.3 and older they weren't read at all. // But this only really matters for CircleCI artifacts, so not too bad. @@ -1491,11 +1545,14 @@ function download($url, $destination) } got_data: ; if ($data == "" || false === file_put_contents($destination, $data)) { + if ($retry) { + return false; + } print_error_and_exit("Error while downloading the installable archive from $url\n"); } echo $okMessage; - return; + return true; next_method: } @@ -1508,11 +1565,14 @@ function download($url, $destination) ); if ($webRequestInvocationStatusCode === 0) { echo $okMessage; - return; + return true; } // Otherwise we attempt other methods } + if ($retry) { + return false; + } echo "Error: Cannot download the installable archive.\n"; echo " One of the following prerequisites must be satisfied:\n"; diff --git a/tooling/bin/generate-final-artifact.sh b/tooling/bin/generate-final-artifact.sh index 77334b6980..10859fdd12 100755 --- a/tooling/bin/generate-final-artifact.sh +++ b/tooling/bin/generate-final-artifact.sh @@ -11,6 +11,69 @@ tmp_folder_final=$tmp_folder/final architectures=(x86_64 aarch64) +php_apis=(20190902 20200930 20210902 20220829 20230831 20240924) +if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then + php_apis+=(20151012 20160303 20170718 20180731) +fi + +targets=(unknown-linux-gnu alpine-linux-musl) +if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then + targets+=(windows) +fi + +configs=("" -zts -debug -debug-zts) + +ln_with_dir() { + mkdir -p $(dirname $2) + ln $1 $2 +} + +for architecture in "${architectures[@]}"; do + for php_api in "${php_apis[@]}"; do + for full_target in "${targets[@]}"; do + target=${full_target#*-} + alpine=$(if [[ $target == "linux-musl" ]]; then echo -alpine; fi) + ext=$([[ $target == "windows" ]] && echo dll || echo so) + for config in "${configs[@]}"; do + ddtrace_ext_path=./extensions_${architecture}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace-${php_api}${alpine}${config}.${ext} + if [[ -f ${ddtrace_ext_path} ]]; then + rm -rf $tmp_folder + mkdir -p $tmp_folder_final + + trace_base_dir=${tmp_folder_final}/dd-library-php/trace + ln_with_dir ${ddtrace_ext_path} ${trace_base_dir}/ext/${php_api}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace${config}.${ext} + cp -r ./src ${trace_base_dir} + + profiling_ext_path=./datadog-profiling/${architecture}-${full_target}/lib/php/${php_api}/datadog-profiling${config}.${ext} + if [[ -f ${profiling_ext_path} ]]; then + profiling_base_dir=${tmp_folder_final}/dd-library-php/profiling + ln_with_dir ${profiling_ext_path} ${profiling_base_dir}/ext/${php_api}/datadog-profiling${config}.${ext} + + # Licenses + ln \ + ./profiling/LICENSE* \ + ./profiling/NOTICE \ + ${profiling_base_dir}/ + fi + + appsec_ext_path=./appsec_${architecture}/ddappsec-${php_api}${alpine}${config}.${ext} + if [[ -f ${appsec_ext_path} ]]; then + appsec_base_dir=${tmp_folder_final}/dd-library-php/appsec + ln_with_dir ${appsec_ext_path} ${appsec_base_dir}/ext/$php_api/ddappsec${config}.${ext} + ln_with_dir ./appsec_${architecture}/libddappsec-helper.so ${appsec_base_dir}/lib/libddappsec-helper.so + ln_with_dir ./appsec_${architecture}/recommended.json ${appsec_base_dir}/etc/recommended.json + fi + + echo "$release_version" > ${tmp_folder_final}/dd-library-php/VERSION + tar -czv \ + -f ${packages_build_dir}/dd-library-php-${release_version}-$architecture-$target-${php_api}${config}.tar.gz \ + -C ${tmp_folder_final} . --owner=0 --group=0 + fi + done + done + done +done + for architecture in "${architectures[@]}"; do tmp_folder_final_gnu=$tmp_folder_final/$architecture-linux-gnu tmp_folder_final_musl=$tmp_folder_final/$architecture-linux-musl @@ -33,10 +96,6 @@ for architecture in "${architectures[@]}"; do tmp_folder_final_musl_trace=$tmp_folder_final_musl/dd-library-php/trace tmp_folder_final_windows_trace=$tmp_folder_final_windows/dd-library-php/trace - php_apis=(20190902 20200930 20210902 20220829 20230831 20240924) - if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then - php_apis+=(20151012 20160303 20170718 20180731) - fi for php_api in "${php_apis[@]}"; do mkdir -p ${tmp_folder_final_gnu_trace}/ext/$php_api ${tmp_folder_final_musl_trace}/ext/$php_api; if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then @@ -44,13 +103,11 @@ for architecture in "${architectures[@]}"; do ln ./extensions_${architecture}/ddtrace-$php_api-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-zts.so; ln ./extensions_${architecture}/ddtrace-$php_api-debug.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug.so; ln ./extensions_${architecture}/ddtrace-$php_api-alpine.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace.so; - if [[ ${php_api} -ge 20151012 ]]; then # zts on alpine starting 7.0 - ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so; - fi + ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so; if [[ ${php_api} -ge 20170718 && $architecture == "x86_64" ]]; then # Windows support starts on 7.2 mkdir -p ${tmp_folder_final_windows_trace}/ext/$php_api; - ln ./extensions_windows_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll; - ln ./extensions_windows_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll; + ln ./extensions_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll; + ln ./extensions_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll; fi else ln ./extensions_${architecture}/ddtrace-$php_api-debug-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug-zts.so; From a9501c47fddf8ff5af86ee28e0c7bc612f6e5fdf Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Fri, 15 Nov 2024 13:58:17 +0100 Subject: [PATCH 2/2] Fix potential inf loop --- datadog-setup.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/datadog-setup.php b/datadog-setup.php index ac843d617c..42ea577660 100644 --- a/datadog-setup.php +++ b/datadog-setup.php @@ -1493,9 +1493,11 @@ function download($url, $destination, $retry = false) $progress_counter = 0; $return = curl_exec($ch); - $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode == 404) { - return false; + if ($retry) { + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($httpCode == 404) { + return false; + } } curl_close($ch);