diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 03107b6d4db..4195bc1dbf4 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -3620,8 +3620,8 @@ jobs: name: Setup docker container shell: powershell.exe command: | - mkdir extensions_windows_x86_64 - mkdir extensions_windows_x86_64_debugsymbols + mkdir extensions_x86_64 + mkdir extensions_x86_64_debugsymbols # No DNS for you by default in circleci docker containers docker network create -d "nat" -o com.docker.network.windowsshim.dnsservers="1.1.1.1" net docker run -v ${pwd}:C:\Users\ContainerAdministrator\app -d --network net --name php << parameters.docker_image >> ping -t localhost @@ -3629,7 +3629,7 @@ jobs: name: Build nts shell: powershell.exe command: | - docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb" + docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb" - run: name: Reuse libdatadog build shell: powershell.exe @@ -3639,10 +3639,10 @@ jobs: name: Build zts shell: powershell.exe command: | - docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb" + docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb" - persist_to_workspace: root: '.' - paths: [ './extensions_windows_x86_64', './extensions_windows_x86_64_debugsymbols' ] + paths: [ './extensions_x86_64', './extensions_x86_64_debugsymbols' ] compile_appsec_extension_centos: working_directory: ~/datadog diff --git a/Makefile b/Makefile index ab10335e0f7..2b3f5ed405b 100644 --- a/Makefile +++ b/Makefile @@ -491,7 +491,7 @@ build_pecl_package: tooling/bin/pecl-build $${FILES//$${BUILD_DIR}/} dbgsym.tar.gz: - $(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_windows_x86_64_debugsymbols --owner=0 --group=0) + $(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_x86_64_debugsymbols --owner=0 --group=0) installer_packages: .apk.x86_64 .apk.aarch64 .rpm.x86_64 .rpm.aarch64 .deb.x86_64 .deb.arm64 .tar.gz.x86_64 .tar.gz.aarch64 bundle.tar.gz dbgsym.tar.gz tar --use-compress-program=pigz --exclude='dd-library-php-ssi-*' -cf packages.tar.gz $(PACKAGES_BUILD_DIR) --owner=0 --group=0 diff --git a/datadog-setup.php b/datadog-setup.php index 051aef8ca15..ac843d617cd 100644 --- a/datadog-setup.php +++ b/datadog-setup.php @@ -95,10 +95,11 @@ function print_help() option can be provided multiple times. --install-dir Install to a specific directory. Default: '$installdir' --uninstall Uninstall the library from the specified binaries. + --file Path to a dd-library-php-*.tar.gz file. Can be used for offline installation. --extension-dir Specify the extension directory. Default: PHP's extension directory. --ini Specify the INI file to use. Default: /98-ddtrace.ini --enable-appsec Enable the application security monitoring module. - --enable-profiling Enable the profiling module. + --enable-profiling Enable the profiling module. -d setting[=value] Used in conjunction with `config ` command to specify the INI setting to get or set. @@ -462,9 +463,58 @@ function install($options) $selectedBinaries = require_binaries_or_exit($options); $interactive = empty($options[OPT_PHP_BIN]); + $commandExtensionSuffixes = []; + $downloadVersions = []; + foreach ($selectedBinaries as $command => $fullPath) { + $binaryForLog = ($command === $fullPath) ? $fullPath : "$command ($fullPath)"; + echo "Checking for binary: $binaryForLog\n"; + + check_php_ext_prerequisite_or_exit($fullPath, 'json'); + + $phpProperties = ini_values($fullPath); + if (!isset($phpProperties[INI_SCANDIR])) { + if (!isset($phpProperties[INI_MAIN])) { + if (IS_WINDOWS) { + $phpProperties[INI_MAIN] = dirname($fullPath) . "/php.ini"; + } else { + print_error_and_exit( + "It is not possible to perform installation on this " + . "system because there is no scan directory and no " + . "configuration file loaded." + ); + } + } + + print_warning( + "Performing an installation without a scan directory may " + . "result in fragile installations that are broken by normal " + . "system upgrades. It is advisable to use the configure " + . "switch --with-config-file-scan-dir when building PHP." + ); + } + + // Suffix (zts/debug) + $extensionSuffix = ''; + if (is_truthy($phpProperties[IS_DEBUG])) { + $extensionSuffix .= '-debug'; + } + if (is_truthy($phpProperties[THREAD_SAFETY])) { + $extensionSuffix .= '-zts'; + } + + $commandExtensionSuffixes[$command] = $extensionSuffix; + + $extensionVersion = $phpProperties[PHP_API]; + $downloadVersions["$extensionVersion$extensionSuffix"] = true; + } + + $tar_gz_suffix = ""; + if (count($downloadVersions) === 1) { + $tar_gz_suffix = "-" . key($downloadVersions); + } + // Preparing clean tmp folder to extract files $tmpDir = sys_get_temp_dir() . '/dd-install'; - $tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}.tar.gz"; $tmpArchiveRoot = $tmpDir . '/dd-library-php'; $tmpArchiveTraceRoot = $tmpDir . '/dd-library-php/trace'; $tmpArchiveAppsecRoot = $tmpDir . '/dd-library-php/appsec'; @@ -490,8 +540,14 @@ function install($options) print_warning('--' . OPT_FILE . ' option is intended for internal usage and can be removed without notice'); $tmpDirTarGz = $options[OPT_FILE]; } else { - $url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}.tar.gz"; - download($url, $tmpDirTarGz); + for (;;) { + $url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}{$tar_gz_suffix}.tar.gz"; + $tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}{$tar_gz_suffix}.tar.gz"; + if (download($url, $tmpDirTarGz, $tar_gz_suffix != "")) { + break; + } + $tar_gz_suffix = ""; // retry with the full archive if the original download failed + } } if (!IS_WINDOWS || `where tar 2> nul` !== null) { execute_or_exit( @@ -574,15 +630,7 @@ function install($options) // Copying the extension $extensionVersion = $phpProperties[PHP_API]; - - // Suffix (zts/debug/alpine) - $extensionSuffix = ''; - if (is_truthy($phpProperties[IS_DEBUG])) { - $extensionSuffix .= '-debug'; - } - if (is_truthy($phpProperties[THREAD_SAFETY])) { - $extensionSuffix .= '-zts'; - } + $extensionSuffix = $commandExtensionSuffixes[$command]; $extDir = isset($options[OPT_EXTENSION_DIR]) ? $options[OPT_EXTENSION_DIR] : $phpProperties[EXTENSION_DIR]; echo "Installing extension to $extDir\n"; @@ -1416,7 +1464,7 @@ function execute_or_exit($exitMessage, $command) * @param string $url * @param string $destination */ -function download($url, $destination) +function download($url, $destination, $retry = false) { echo "Downloading installable archive from $url.\n"; echo "This operation might take a while.\n"; @@ -1444,12 +1492,18 @@ function download($url, $destination) curl_setopt($ch, CURLOPT_NOPROGRESS, false); $progress_counter = 0; $return = curl_exec($ch); + + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($httpCode == 404) { + return false; + } + curl_close($ch); fclose($fp); if (false !== $return) { echo $okMessage; - return; + return true; } // Otherwise we attempt other methods } @@ -1461,20 +1515,20 @@ function download($url, $destination) if (!IS_WINDOWS && false !== exec('curl --version', $output, $statusCode) && $statusCode === 0) { $curlInvocationStatusCode = 0; system( - 'curl -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url), + 'curl -f -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url), $curlInvocationStatusCode ); if ($curlInvocationStatusCode === 0) { echo $okMessage; - return; + return true; } // Otherwise we attempt other methods } // file_get_contents if (is_truthy(ini_get('allow_url_fopen')) && extension_loaded('openssl')) { - ini_set("memory_limit", "1G"); // increase memory limit otherwise we may run OOM here. + ini_set("memory_limit", "2G"); // increase memory limit otherwise we may run OOM here. $data = @file_get_contents($url); // PHP doesn't like too long location headers, and on PHP 7.3 and older they weren't read at all. // But this only really matters for CircleCI artifacts, so not too bad. @@ -1491,11 +1545,14 @@ function download($url, $destination) } got_data: ; if ($data == "" || false === file_put_contents($destination, $data)) { + if ($retry) { + return false; + } print_error_and_exit("Error while downloading the installable archive from $url\n"); } echo $okMessage; - return; + return true; next_method: } @@ -1508,11 +1565,14 @@ function download($url, $destination) ); if ($webRequestInvocationStatusCode === 0) { echo $okMessage; - return; + return true; } // Otherwise we attempt other methods } + if ($retry) { + return false; + } echo "Error: Cannot download the installable archive.\n"; echo " One of the following prerequisites must be satisfied:\n"; diff --git a/tooling/bin/generate-final-artifact.sh b/tooling/bin/generate-final-artifact.sh index 77334b69808..10859fdd12d 100755 --- a/tooling/bin/generate-final-artifact.sh +++ b/tooling/bin/generate-final-artifact.sh @@ -11,6 +11,69 @@ tmp_folder_final=$tmp_folder/final architectures=(x86_64 aarch64) +php_apis=(20190902 20200930 20210902 20220829 20230831 20240924) +if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then + php_apis+=(20151012 20160303 20170718 20180731) +fi + +targets=(unknown-linux-gnu alpine-linux-musl) +if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then + targets+=(windows) +fi + +configs=("" -zts -debug -debug-zts) + +ln_with_dir() { + mkdir -p $(dirname $2) + ln $1 $2 +} + +for architecture in "${architectures[@]}"; do + for php_api in "${php_apis[@]}"; do + for full_target in "${targets[@]}"; do + target=${full_target#*-} + alpine=$(if [[ $target == "linux-musl" ]]; then echo -alpine; fi) + ext=$([[ $target == "windows" ]] && echo dll || echo so) + for config in "${configs[@]}"; do + ddtrace_ext_path=./extensions_${architecture}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace-${php_api}${alpine}${config}.${ext} + if [[ -f ${ddtrace_ext_path} ]]; then + rm -rf $tmp_folder + mkdir -p $tmp_folder_final + + trace_base_dir=${tmp_folder_final}/dd-library-php/trace + ln_with_dir ${ddtrace_ext_path} ${trace_base_dir}/ext/${php_api}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace${config}.${ext} + cp -r ./src ${trace_base_dir} + + profiling_ext_path=./datadog-profiling/${architecture}-${full_target}/lib/php/${php_api}/datadog-profiling${config}.${ext} + if [[ -f ${profiling_ext_path} ]]; then + profiling_base_dir=${tmp_folder_final}/dd-library-php/profiling + ln_with_dir ${profiling_ext_path} ${profiling_base_dir}/ext/${php_api}/datadog-profiling${config}.${ext} + + # Licenses + ln \ + ./profiling/LICENSE* \ + ./profiling/NOTICE \ + ${profiling_base_dir}/ + fi + + appsec_ext_path=./appsec_${architecture}/ddappsec-${php_api}${alpine}${config}.${ext} + if [[ -f ${appsec_ext_path} ]]; then + appsec_base_dir=${tmp_folder_final}/dd-library-php/appsec + ln_with_dir ${appsec_ext_path} ${appsec_base_dir}/ext/$php_api/ddappsec${config}.${ext} + ln_with_dir ./appsec_${architecture}/libddappsec-helper.so ${appsec_base_dir}/lib/libddappsec-helper.so + ln_with_dir ./appsec_${architecture}/recommended.json ${appsec_base_dir}/etc/recommended.json + fi + + echo "$release_version" > ${tmp_folder_final}/dd-library-php/VERSION + tar -czv \ + -f ${packages_build_dir}/dd-library-php-${release_version}-$architecture-$target-${php_api}${config}.tar.gz \ + -C ${tmp_folder_final} . --owner=0 --group=0 + fi + done + done + done +done + for architecture in "${architectures[@]}"; do tmp_folder_final_gnu=$tmp_folder_final/$architecture-linux-gnu tmp_folder_final_musl=$tmp_folder_final/$architecture-linux-musl @@ -33,10 +96,6 @@ for architecture in "${architectures[@]}"; do tmp_folder_final_musl_trace=$tmp_folder_final_musl/dd-library-php/trace tmp_folder_final_windows_trace=$tmp_folder_final_windows/dd-library-php/trace - php_apis=(20190902 20200930 20210902 20220829 20230831 20240924) - if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then - php_apis+=(20151012 20160303 20170718 20180731) - fi for php_api in "${php_apis[@]}"; do mkdir -p ${tmp_folder_final_gnu_trace}/ext/$php_api ${tmp_folder_final_musl_trace}/ext/$php_api; if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then @@ -44,13 +103,11 @@ for architecture in "${architectures[@]}"; do ln ./extensions_${architecture}/ddtrace-$php_api-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-zts.so; ln ./extensions_${architecture}/ddtrace-$php_api-debug.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug.so; ln ./extensions_${architecture}/ddtrace-$php_api-alpine.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace.so; - if [[ ${php_api} -ge 20151012 ]]; then # zts on alpine starting 7.0 - ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so; - fi + ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so; if [[ ${php_api} -ge 20170718 && $architecture == "x86_64" ]]; then # Windows support starts on 7.2 mkdir -p ${tmp_folder_final_windows_trace}/ext/$php_api; - ln ./extensions_windows_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll; - ln ./extensions_windows_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll; + ln ./extensions_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll; + ln ./extensions_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll; fi else ln ./extensions_${architecture}/ddtrace-$php_api-debug-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug-zts.so;