Create many tiny artifacts for faster download (#2951)

* Create many tiny artifacts for faster download

And not escaping the constraints many systems have on /tmp size.
If someone installs multiple php versions at once, we still download the full package.

The full package also remains available for offline installations of multiple targets for now.

Signed-off-by: Bob Weinand <[email protected]>

* Fix potential inf loop

---------

Signed-off-by: Bob Weinand <[email protected]>

Author: bwoebi

.circleci/continue_config.yml

@@ -3620,16 +3620,16 @@ jobs:
           name: Setup docker container
           shell: powershell.exe
           command: |
-            mkdir extensions_windows_x86_64
-            mkdir extensions_windows_x86_64_debugsymbols
+            mkdir extensions_x86_64
+            mkdir extensions_x86_64_debugsymbols
             # No DNS for you by default in circleci docker containers
             docker network create -d "nat" -o com.docker.network.windowsshim.dnsservers="1.1.1.1" net
             docker run -v ${pwd}:C:\Users\ContainerAdministrator\app -d --network net --name php << parameters.docker_image >> ping -t localhost
       - run:
           name: Build nts
           shell: powershell.exe
           command: |
-            docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb"
+            docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb"
       - run:
           name: Reuse libdatadog build
           shell: powershell.exe
@@ -3639,10 +3639,10 @@ jobs:
           name: Build zts
           shell: powershell.exe
           command: |
-            docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb"
+            docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb"
       - persist_to_workspace:
           root: '.'
-          paths: [ './extensions_windows_x86_64', './extensions_windows_x86_64_debugsymbols' ]
+          paths: [ './extensions_x86_64', './extensions_x86_64_debugsymbols' ]
 
   compile_appsec_extension_centos:
     working_directory: ~/datadog

Makefile

@@ -492,7 +492,7 @@ build_pecl_package:
 	tooling/bin/pecl-build $${FILES//$${BUILD_DIR}/}
 
 dbgsym.tar.gz:
-	$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_windows_x86_64_debugsymbols --owner=0 --group=0)
+	$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_x86_64_debugsymbols --owner=0 --group=0)
 
 installer_packages: .apk.x86_64 .apk.aarch64 .rpm.x86_64 .rpm.aarch64 .deb.x86_64 .deb.arm64 .tar.gz.x86_64 .tar.gz.aarch64 bundle.tar.gz dbgsym.tar.gz
 	tar --use-compress-program=pigz --exclude='dd-library-php-ssi-*' -cf packages.tar.gz $(PACKAGES_BUILD_DIR) --owner=0 --group=0

datadog-setup.php

@@ -95,10 +95,11 @@ function print_help()
                                 option can be provided multiple times.
     --install-dir <path>        Install to a specific directory. Default: '$installdir'
     --uninstall                 Uninstall the library from the specified binaries.
+    --file <path to .tar.gz>    Path to a dd-library-php-*.tar.gz file. Can be used for offline installation.
     --extension-dir <path>      Specify the extension directory. Default: PHP's extension directory.
     --ini <path>                Specify the INI file to use. Default: <ini-dir>/98-ddtrace.ini
     --enable-appsec             Enable the application security monitoring module.
-    --enable-profiling           Enable the profiling module.
+    --enable-profiling          Enable the profiling module.
     -d setting[=value]          Used in conjunction with `config <set|get>`
                                 command to specify the INI setting to get or set.
 
@@ -462,9 +463,58 @@ function install($options)
     $selectedBinaries = require_binaries_or_exit($options);
     $interactive = empty($options[OPT_PHP_BIN]);
 
+    $commandExtensionSuffixes = [];
+    $downloadVersions = [];
+    foreach ($selectedBinaries as $command => $fullPath) {
+        $binaryForLog = ($command === $fullPath) ? $fullPath : "$command ($fullPath)";
+        echo "Checking for binary: $binaryForLog\n";
+
+        check_php_ext_prerequisite_or_exit($fullPath, 'json');
+
+        $phpProperties = ini_values($fullPath);
+        if (!isset($phpProperties[INI_SCANDIR])) {
+            if (!isset($phpProperties[INI_MAIN])) {
+                if (IS_WINDOWS) {
+                    $phpProperties[INI_MAIN] = dirname($fullPath) . "/php.ini";
+                } else {
+                    print_error_and_exit(
+                        "It is not possible to perform installation on this "
+                        . "system because there is no scan directory and no "
+                        . "configuration file loaded."
+                    );
+                }
+            }
+
+            print_warning(
+                "Performing an installation without a scan directory may "
+                . "result in fragile installations that are broken by normal "
+                . "system upgrades. It is advisable to use the configure "
+                . "switch --with-config-file-scan-dir when building PHP."
+            );
+        }
+
+        // Suffix (zts/debug)
+        $extensionSuffix = '';
+        if (is_truthy($phpProperties[IS_DEBUG])) {
+            $extensionSuffix .= '-debug';
+        }
+        if (is_truthy($phpProperties[THREAD_SAFETY])) {
+            $extensionSuffix .= '-zts';
+        }
+
+        $commandExtensionSuffixes[$command] = $extensionSuffix;
+
+        $extensionVersion = $phpProperties[PHP_API];
+        $downloadVersions["$extensionVersion$extensionSuffix"] = true;
+    }
+
+    $tar_gz_suffix = "";
+    if (count($downloadVersions) === 1) {
+        $tar_gz_suffix = "-" . key($downloadVersions);
+    }
+
     // Preparing clean tmp folder to extract files
     $tmpDir = sys_get_temp_dir() . '/dd-install';
-    $tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}.tar.gz";
     $tmpArchiveRoot = $tmpDir . '/dd-library-php';
     $tmpArchiveTraceRoot = $tmpDir . '/dd-library-php/trace';
     $tmpArchiveAppsecRoot = $tmpDir . '/dd-library-php/appsec';
@@ -490,8 +540,14 @@ function install($options)
         print_warning('--' . OPT_FILE . ' option is intended for internal usage and can be removed without notice');
         $tmpDirTarGz = $options[OPT_FILE];
     } else {
-        $url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}.tar.gz";
-        download($url, $tmpDirTarGz);
+        for (;;) {
+            $url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}{$tar_gz_suffix}.tar.gz";
+            $tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}{$tar_gz_suffix}.tar.gz";
+            if (download($url, $tmpDirTarGz, $tar_gz_suffix != "")) {
+                break;
+            }
+            $tar_gz_suffix = ""; // retry with the full archive if the original download failed
+        }
     }
     if (!IS_WINDOWS || `where tar 2> nul` !== null) {
         execute_or_exit(
@@ -574,15 +630,7 @@ function install($options)
 
         // Copying the extension
         $extensionVersion = $phpProperties[PHP_API];
-
-        // Suffix (zts/debug/alpine)
-        $extensionSuffix = '';
-        if (is_truthy($phpProperties[IS_DEBUG])) {
-            $extensionSuffix .= '-debug';
-        }
-        if (is_truthy($phpProperties[THREAD_SAFETY])) {
-            $extensionSuffix .= '-zts';
-        }
+        $extensionSuffix = $commandExtensionSuffixes[$command];
 
         $extDir = isset($options[OPT_EXTENSION_DIR]) ? $options[OPT_EXTENSION_DIR] : $phpProperties[EXTENSION_DIR];
         echo "Installing extension to $extDir\n";
@@ -1416,7 +1464,7 @@ function execute_or_exit($exitMessage, $command)
  * @param string $url
  * @param string $destination
  */
-function download($url, $destination)
+function download($url, $destination, $retry = false)
 {
     echo "Downloading installable archive from $url.\n";
     echo "This operation might take a while.\n";
@@ -1444,12 +1492,20 @@ function download($url, $destination)
         curl_setopt($ch, CURLOPT_NOPROGRESS, false);
         $progress_counter = 0;
         $return = curl_exec($ch);
+
+        if ($retry) {
+            $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+            if ($httpCode == 404) {
+                return false;
+            }
+        }
+
         curl_close($ch);
         fclose($fp);
 
         if (false !== $return) {
             echo $okMessage;
-            return;
+            return true;
         }
         // Otherwise we attempt other methods
     }
@@ -1461,20 +1517,20 @@ function download($url, $destination)
     if (!IS_WINDOWS && false !== exec('curl --version', $output, $statusCode) && $statusCode === 0) {
         $curlInvocationStatusCode = 0;
         system(
-            'curl -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url),
+            'curl -f -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url),
             $curlInvocationStatusCode
         );
 
         if ($curlInvocationStatusCode === 0) {
             echo $okMessage;
-            return;
+            return true;
         }
         // Otherwise we attempt other methods
     }
 
     // file_get_contents
     if (is_truthy(ini_get('allow_url_fopen')) && extension_loaded('openssl')) {
-        ini_set("memory_limit", "1G"); // increase memory limit otherwise we may run OOM here.
+        ini_set("memory_limit", "2G"); // increase memory limit otherwise we may run OOM here.
         $data = @file_get_contents($url);
         // PHP doesn't like too long location headers, and on PHP 7.3 and older they weren't read at all.
         // But this only really matters for CircleCI artifacts, so not too bad.
@@ -1491,11 +1547,14 @@ function download($url, $destination)
         }
         got_data: ;
         if ($data == "" || false === file_put_contents($destination, $data)) {
+            if ($retry) {
+                return false;
+            }
             print_error_and_exit("Error while downloading the installable archive from $url\n");
         }
 
         echo $okMessage;
-        return;
+        return true;
 
         next_method:
     }
@@ -1508,11 +1567,14 @@ function download($url, $destination)
         );
         if ($webRequestInvocationStatusCode === 0) {
             echo $okMessage;
-            return;
+            return true;
         }
         // Otherwise we attempt other methods
     }
 
+    if ($retry) {
+        return false;
+    }
 
     echo "Error: Cannot download the installable archive.\n";
     echo "  One of the following prerequisites must be satisfied:\n";

tooling/bin/generate-final-artifact.sh

@@ -11,6 +11,69 @@ tmp_folder_final=$tmp_folder/final
 
 architectures=(x86_64 aarch64)
 
+php_apis=(20190902 20200930 20210902 20220829 20230831 20240924)
+if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
+    php_apis+=(20151012 20160303 20170718 20180731)
+fi
+
+targets=(unknown-linux-gnu alpine-linux-musl)
+if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
+    targets+=(windows)
+fi
+
+configs=("" -zts -debug -debug-zts)
+
+ln_with_dir() {
+    mkdir -p $(dirname $2)
+    ln $1 $2
+}
+
+for architecture in "${architectures[@]}"; do
+    for php_api in "${php_apis[@]}"; do
+        for full_target in "${targets[@]}"; do
+            target=${full_target#*-}
+            alpine=$(if [[ $target == "linux-musl" ]]; then echo -alpine; fi)
+            ext=$([[ $target == "windows" ]] && echo dll || echo so)
+            for config in "${configs[@]}"; do
+                ddtrace_ext_path=./extensions_${architecture}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace-${php_api}${alpine}${config}.${ext}
+                if [[ -f ${ddtrace_ext_path} ]]; then
+                    rm -rf $tmp_folder
+                    mkdir -p $tmp_folder_final
+
+                    trace_base_dir=${tmp_folder_final}/dd-library-php/trace
+                    ln_with_dir ${ddtrace_ext_path} ${trace_base_dir}/ext/${php_api}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace${config}.${ext}
+                    cp -r ./src ${trace_base_dir}
+
+                    profiling_ext_path=./datadog-profiling/${architecture}-${full_target}/lib/php/${php_api}/datadog-profiling${config}.${ext}
+                    if [[ -f ${profiling_ext_path} ]]; then
+                        profiling_base_dir=${tmp_folder_final}/dd-library-php/profiling
+                        ln_with_dir ${profiling_ext_path} ${profiling_base_dir}/ext/${php_api}/datadog-profiling${config}.${ext}
+
+                        # Licenses
+                        ln \
+                            ./profiling/LICENSE* \
+                            ./profiling/NOTICE \
+                            ${profiling_base_dir}/
+                    fi
+
+                    appsec_ext_path=./appsec_${architecture}/ddappsec-${php_api}${alpine}${config}.${ext}
+                    if [[ -f ${appsec_ext_path} ]]; then
+                        appsec_base_dir=${tmp_folder_final}/dd-library-php/appsec
+                        ln_with_dir ${appsec_ext_path} ${appsec_base_dir}/ext/$php_api/ddappsec${config}.${ext}
+                        ln_with_dir ./appsec_${architecture}/libddappsec-helper.so ${appsec_base_dir}/lib/libddappsec-helper.so
+                        ln_with_dir ./appsec_${architecture}/recommended.json ${appsec_base_dir}/etc/recommended.json
+                    fi
+
+                    echo "$release_version" > ${tmp_folder_final}/dd-library-php/VERSION
+                    tar -czv \
+                        -f ${packages_build_dir}/dd-library-php-${release_version}-$architecture-$target-${php_api}${config}.tar.gz \
+                        -C ${tmp_folder_final} . --owner=0 --group=0
+                fi
+            done
+        done
+    done
+done
+
 for architecture in "${architectures[@]}"; do
     tmp_folder_final_gnu=$tmp_folder_final/$architecture-linux-gnu
     tmp_folder_final_musl=$tmp_folder_final/$architecture-linux-musl
@@ -33,24 +96,18 @@ for architecture in "${architectures[@]}"; do
     tmp_folder_final_musl_trace=$tmp_folder_final_musl/dd-library-php/trace
     tmp_folder_final_windows_trace=$tmp_folder_final_windows/dd-library-php/trace
 
-    php_apis=(20190902 20200930 20210902 20220829 20230831 20240924)
-    if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
-        php_apis+=(20151012 20160303 20170718 20180731)
-    fi
     for php_api in "${php_apis[@]}"; do
         mkdir -p ${tmp_folder_final_gnu_trace}/ext/$php_api ${tmp_folder_final_musl_trace}/ext/$php_api;
         if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
             ln ./extensions_${architecture}/ddtrace-$php_api.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace.so;
             ln ./extensions_${architecture}/ddtrace-$php_api-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-zts.so;
             ln ./extensions_${architecture}/ddtrace-$php_api-debug.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug.so;
             ln ./extensions_${architecture}/ddtrace-$php_api-alpine.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace.so;
-            if [[ ${php_api} -ge 20151012 ]]; then # zts on alpine starting 7.0
-                ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so;
-            fi
+            ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so;
             if [[ ${php_api} -ge 20170718 && $architecture == "x86_64" ]]; then # Windows support starts on 7.2
                 mkdir -p ${tmp_folder_final_windows_trace}/ext/$php_api;
-                ln ./extensions_windows_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll;
-                ln ./extensions_windows_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll;
+                ln ./extensions_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll;
+                ln ./extensions_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll;
             fi
         else
             ln ./extensions_${architecture}/ddtrace-$php_api-debug-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug-zts.so;