Create many tiny artifacts for faster download

.circleci/continue_config.yml

@@ -3620,16 +3620,16 @@ jobs:
           name: Setup docker container
           shell: powershell.exe
           command: |
-            mkdir extensions_windows_x86_64
-            mkdir extensions_windows_x86_64_debugsymbols
+            mkdir extensions_x86_64
+            mkdir extensions_x86_64_debugsymbols
             # No DNS for you by default in circleci docker containers
             docker network create -d "nat" -o com.docker.network.windowsshim.dnsservers="1.1.1.1" net
             docker run -v ${pwd}:C:\Users\ContainerAdministrator\app -d --network net --name php << parameters.docker_image >> ping -t localhost
       - run:
           name: Build nts
           shell: powershell.exe
           command: |
-            docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb"
+            docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb"
       - run:
           name: Reuse libdatadog build
           shell: powershell.exe
@@ -3639,10 +3639,10 @@ jobs:
           name: Build zts
           shell: powershell.exe
           command: |
-            docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb"
+            docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb"
       - persist_to_workspace:
           root: '.'
-          paths: [ './extensions_windows_x86_64', './extensions_windows_x86_64_debugsymbols' ]
+          paths: [ './extensions_x86_64', './extensions_x86_64_debugsymbols' ]
 
   compile_appsec_extension_centos:
     working_directory: ~/datadog

Makefile

@@ -491,7 +491,7 @@ build_pecl_package:
 	tooling/bin/pecl-build $${FILES//$${BUILD_DIR}/}
 
 dbgsym.tar.gz:
-	$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_windows_x86_64_debugsymbols --owner=0 --group=0)
+	$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_x86_64_debugsymbols --owner=0 --group=0)
 
 installer_packages: .apk.x86_64 .apk.aarch64 .rpm.x86_64 .rpm.aarch64 .deb.x86_64 .deb.arm64 .tar.gz.x86_64 .tar.gz.aarch64 bundle.tar.gz dbgsym.tar.gz
 	tar --use-compress-program=pigz --exclude='dd-library-php-ssi-*' -cf packages.tar.gz $(PACKAGES_BUILD_DIR) --owner=0 --group=0

datadog-setup.php

@@ -95,10 +95,11 @@ function print_help()
                                 option can be provided multiple times.
     --install-dir <path>        Install to a specific directory. Default: '$installdir'
     --uninstall                 Uninstall the library from the specified binaries.
+    --file <path to .tar.gz>    Path to a dd-library-php-*.tar.gz file. Can be used for offline installation.
     --extension-dir <path>      Specify the extension directory. Default: PHP's extension directory.
     --ini <path>                Specify the INI file to use. Default: <ini-dir>/98-ddtrace.ini
     --enable-appsec             Enable the application security monitoring module.
-    --enable-profiling           Enable the profiling module.
+    --enable-profiling          Enable the profiling module.
     -d setting[=value]          Used in conjunction with `config <set|get>`
                                 command to specify the INI setting to get or set.
 
@@ -462,9 +463,58 @@ function install($options)
     $selectedBinaries = require_binaries_or_exit($options);
     $interactive = empty($options[OPT_PHP_BIN]);
 
+    $commandExtensionSuffixes = [];
+    $downloadVersions = [];
+    foreach ($selectedBinaries as $command => $fullPath) {
+        $binaryForLog = ($command === $fullPath) ? $fullPath : "$command ($fullPath)";
+        echo "Checking for binary: $binaryForLog\n";
+
+        check_php_ext_prerequisite_or_exit($fullPath, 'json');
+
+        $phpProperties = ini_values($fullPath);
+        if (!isset($phpProperties[INI_SCANDIR])) {
+            if (!isset($phpProperties[INI_MAIN])) {
+                if (IS_WINDOWS) {
+                    $phpProperties[INI_MAIN] = dirname($fullPath) . "/php.ini";
+                } else {
+                    print_error_and_exit(
+                        "It is not possible to perform installation on this "
+                        . "system because there is no scan directory and no "
+                        . "configuration file loaded."
+                    );
+                }
+            }
+
+            print_warning(
+                "Performing an installation without a scan directory may "
+                . "result in fragile installations that are broken by normal "
+                . "system upgrades. It is advisable to use the configure "
+                . "switch --with-config-file-scan-dir when building PHP."
+            );
+        }
+
+        // Suffix (zts/debug)
+        $extensionSuffix = '';
+        if (is_truthy($phpProperties[IS_DEBUG])) {
+            $extensionSuffix .= '-debug';
+        }
+        if (is_truthy($phpProperties[THREAD_SAFETY])) {
+            $extensionSuffix .= '-zts';
+        }
+
+        $commandExtensionSuffixes[$command] = $extensionSuffix;
+
+        $extensionVersion = $phpProperties[PHP_API];
+        $downloadVersions["$extensionVersion$extensionSuffix"] = true;
+    }
+
+    $tar_gz_suffix = "";
+    if (count($downloadVersions) === 1) {
+        $tar_gz_suffix = "-" . key($downloadVersions);
+    }
+
     // Preparing clean tmp folder to extract files
     $tmpDir = sys_get_temp_dir() . '/dd-install';
-    $tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}.tar.gz";
     $tmpArchiveRoot = $tmpDir . '/dd-library-php';
     $tmpArchiveTraceRoot = $tmpDir . '/dd-library-php/trace';
     $tmpArchiveAppsecRoot = $tmpDir . '/dd-library-php/appsec';
@@ -490,8 +540,14 @@ function install($options)
         print_warning('--' . OPT_FILE . ' option is intended for internal usage and can be removed without notice');
         $tmpDirTarGz = $options[OPT_FILE];
     } else {
-        $url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}.tar.gz";
-        download($url, $tmpDirTarGz);
+        for (;;) {
+            $url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}{$tar_gz_suffix}.tar.gz";
+            $tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}{$tar_gz_suffix}.tar.gz";
+            if (download($url, $tmpDirTarGz, $tar_gz_suffix != "")) {
+                break;
+            }
+            $tar_gz_suffix = ""; // retry with the full archive if the original download failed
+        }
     }
     if (!IS_WINDOWS || `where tar 2> nul` !== null) {
         execute_or_exit(
@@ -574,15 +630,7 @@ function install($options)
 
         // Copying the extension
         $extensionVersion = $phpProperties[PHP_API];
-
-        // Suffix (zts/debug/alpine)
-        $extensionSuffix = '';
-        if (is_truthy($phpProperties[IS_DEBUG])) {
-            $extensionSuffix .= '-debug';
-        }
-        if (is_truthy($phpProperties[THREAD_SAFETY])) {
-            $extensionSuffix .= '-zts';
-        }
+        $extensionSuffix = $commandExtensionSuffixes[$command];
 
         $extDir = isset($options[OPT_EXTENSION_DIR]) ? $options[OPT_EXTENSION_DIR] : $phpProperties[EXTENSION_DIR];
         echo "Installing extension to $extDir\n";
@@ -1416,7 +1464,7 @@ function execute_or_exit($exitMessage, $command)
  * @param string $url
  * @param string $destination
  */
-function download($url, $destination)
+function download($url, $destination, $retry = false)
 {
     echo "Downloading installable archive from $url.\n";
     echo "This operation might take a while.\n";
@@ -1444,12 +1492,20 @@ function download($url, $destination)
         curl_setopt($ch, CURLOPT_NOPROGRESS, false);
         $progress_counter = 0;
         $return = curl_exec($ch);
+
+        if ($retry) {
+            $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+            if ($httpCode == 404) {
+                return false;
+            }
+        }
+
         curl_close($ch);
         fclose($fp);
 
         if (false !== $return) {
             echo $okMessage;
-            return;
+            return true;
         }
         // Otherwise we attempt other methods
     }
@@ -1461,20 +1517,20 @@ function download($url, $destination)
     if (!IS_WINDOWS && false !== exec('curl --version', $output, $statusCode) && $statusCode === 0) {
         $curlInvocationStatusCode = 0;
         system(
-            'curl -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url),
+            'curl -f -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url),
             $curlInvocationStatusCode
         );
 
         if ($curlInvocationStatusCode === 0) {
             echo $okMessage;
-            return;
+            return true;
         }
         // Otherwise we attempt other methods
     }
 
     // file_get_contents
     if (is_truthy(ini_get('allow_url_fopen')) && extension_loaded('openssl')) {
-        ini_set("memory_limit", "1G"); // increase memory limit otherwise we may run OOM here.
+        ini_set("memory_limit", "2G"); // increase memory limit otherwise we may run OOM here.
         $data = @file_get_contents($url);
         // PHP doesn't like too long location headers, and on PHP 7.3 and older they weren't read at all.
         // But this only really matters for CircleCI artifacts, so not too bad.
@@ -1491,11 +1547,14 @@ function download($url, $destination)
         }
         got_data: ;
         if ($data == "" || false === file_put_contents($destination, $data)) {
+            if ($retry) {
+                return false;
+            }
             print_error_and_exit("Error while downloading the installable archive from $url\n");
         }
 
         echo $okMessage;
-        return;
+        return true;
 
         next_method:
     }
@@ -1508,11 +1567,14 @@ function download($url, $destination)
         );
         if ($webRequestInvocationStatusCode === 0) {
             echo $okMessage;
-            return;
+            return true;
         }
         // Otherwise we attempt other methods
     }
 
+    if ($retry) {
+        return false;
+    }
 
     echo "Error: Cannot download the installable archive.\n";
     echo "  One of the following prerequisites must be satisfied:\n";

tooling/bin/generate-final-artifact.sh

@@ -11,6 +11,69 @@ tmp_folder_final=$tmp_folder/final
 
 architectures=(x86_64 aarch64)
 
+php_apis=(20190902 20200930 20210902 20220829 20230831 20240924)
+if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
+    php_apis+=(20151012 20160303 20170718 20180731)
+fi
+
+targets=(unknown-linux-gnu alpine-linux-musl)
+if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
+    targets+=(windows)
+fi
+
+configs=("" -zts -debug -debug-zts)
+
+ln_with_dir() {
+    mkdir -p $(dirname $2)
+    ln $1 $2
+}
+
+for architecture in "${architectures[@]}"; do
+    for php_api in "${php_apis[@]}"; do
+        for full_target in "${targets[@]}"; do
+            target=${full_target#*-}
+            alpine=$(if [[ $target == "linux-musl" ]]; then echo -alpine; fi)
+            ext=$([[ $target == "windows" ]] && echo dll || echo so)
+            for config in "${configs[@]}"; do
+                ddtrace_ext_path=./extensions_${architecture}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace-${php_api}${alpine}${config}.${ext}
+                if [[ -f ${ddtrace_ext_path} ]]; then
+                    rm -rf $tmp_folder
+                    mkdir -p $tmp_folder_final
+
+                    trace_base_dir=${tmp_folder_final}/dd-library-php/trace
+                    ln_with_dir ${ddtrace_ext_path} ${trace_base_dir}/ext/${php_api}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace${config}.${ext}
+                    cp -r ./src ${trace_base_dir}
+
+                    profiling_ext_path=./datadog-profiling/${architecture}-${full_target}/lib/php/${php_api}/datadog-profiling${config}.${ext}
+                    if [[ -f ${profiling_ext_path} ]]; then
+                        profiling_base_dir=${tmp_folder_final}/dd-library-php/profiling
+                        ln_with_dir ${profiling_ext_path} ${profiling_base_dir}/ext/${php_api}/datadog-profiling${config}.${ext}
+
+                        # Licenses
+                        ln \
+                            ./profiling/LICENSE* \
+                            ./profiling/NOTICE \
+                            ${profiling_base_dir}/
+                    fi
+
+                    appsec_ext_path=./appsec_${architecture}/ddappsec-${php_api}${alpine}${config}.${ext}
+                    if [[ -f ${appsec_ext_path} ]]; then
+                        appsec_base_dir=${tmp_folder_final}/dd-library-php/appsec
+                        ln_with_dir ${appsec_ext_path} ${appsec_base_dir}/ext/$php_api/ddappsec${config}.${ext}
+                        ln_with_dir ./appsec_${architecture}/libddappsec-helper.so ${appsec_base_dir}/lib/libddappsec-helper.so
+                        ln_with_dir ./appsec_${architecture}/recommended.json ${appsec_base_dir}/etc/recommended.json
+                    fi
+
+                    echo "$release_version" > ${tmp_folder_final}/dd-library-php/VERSION
+                    tar -czv \
+                        -f ${packages_build_dir}/dd-library-php-${release_version}-$architecture-$target-${php_api}${config}.tar.gz \
+                        -C ${tmp_folder_final} . --owner=0 --group=0
+                fi
+            done
+        done
+    done
+done
+
 for architecture in "${architectures[@]}"; do
     tmp_folder_final_gnu=$tmp_folder_final/$architecture-linux-gnu
     tmp_folder_final_musl=$tmp_folder_final/$architecture-linux-musl
@@ -33,24 +96,18 @@ for architecture in "${architectures[@]}"; do
     tmp_folder_final_musl_trace=$tmp_folder_final_musl/dd-library-php/trace
     tmp_folder_final_windows_trace=$tmp_folder_final_windows/dd-library-php/trace
 
-    php_apis=(20190902 20200930 20210902 20220829 20230831 20240924)
-    if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
-        php_apis+=(20151012 20160303 20170718 20180731)
-    fi
     for php_api in "${php_apis[@]}"; do
         mkdir -p ${tmp_folder_final_gnu_trace}/ext/$php_api ${tmp_folder_final_musl_trace}/ext/$php_api;
         if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
             ln ./extensions_${architecture}/ddtrace-$php_api.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace.so;
             ln ./extensions_${architecture}/ddtrace-$php_api-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-zts.so;
             ln ./extensions_${architecture}/ddtrace-$php_api-debug.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug.so;
             ln ./extensions_${architecture}/ddtrace-$php_api-alpine.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace.so;
-            if [[ ${php_api} -ge 20151012 ]]; then # zts on alpine starting 7.0
-                ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so;
-            fi
+            ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so;
             if [[ ${php_api} -ge 20170718 && $architecture == "x86_64" ]]; then # Windows support starts on 7.2
                 mkdir -p ${tmp_folder_final_windows_trace}/ext/$php_api;
-                ln ./extensions_windows_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll;
-                ln ./extensions_windows_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll;
+                ln ./extensions_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll;
+                ln ./extensions_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll;
             fi
         else
             ln ./extensions_${architecture}/ddtrace-$php_api-debug-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug-zts.so;