WIP: Create a unified build for (swift-)foundation-tests

We can’t add the tests to the general unified build because they use testable imports and building a package with testable imports also re-builds all of its dependencies with testable imports (swiftlang/swift-package-manager#8344) and is thus incompatible with the other package’s unified build. But we should still be able to share build products between corelibs-foundation tests and swift-foundation-tests.

Author: ahoppen

utils/build_swift/resources/SwiftPM-Unified-Build.xcworkspace/contents.xcworkspacedata

@@ -893,9 +893,8 @@ def _execute_action(self, action_name):
     def execute_product_build_steps(self, product_class, host_target):
         product_source = product_class.product_source_name()
         product_name = product_class.product_name()
-        if product_class.is_swiftpm_unified_build_product():
-            build_dir = self.workspace.swiftpm_unified_build_dir(
-                host_target)
+        if arena := product_class.swiftpm_unified_build_product_arena():
+            build_dir = self.workspace.swiftpm_unified_build_dir(arena, host_target)
         else:
             build_dir = self.workspace.build_dir(
                 host_target, product_name)

utils/swift_build_support/swift_build_support/build_script_invocation.py

@@ -12,6 +12,8 @@
 
 import os
 
+from build_swift.build_swift.constants import MULTIROOT_DATA_FILE_PATH
+
 from . import cmark
 from . import foundation
 from . import libcxx
@@ -35,6 +37,10 @@ def is_build_script_impl_product(cls):
     def is_before_build_script_impl_product(cls):
         return False
 
+    @classmethod
+    def swiftpm_unified_build_product_arena(cls):
+        return 'foundationtests'
+
     @classmethod
     def is_ignore_install_all_product(cls):
         return True
@@ -68,16 +74,51 @@ def test(self, host_target):
             'lib',
             'swift'
         )
+        plutil_cmd = [
+            swift_exec,
+            'build',
+            '--toolchain', self.install_toolchain_path(host_target),
+            '--configuration', self.configuration(),
+            '--scratch-path', self.build_dir,
+            '--package-path', package_path,
+            '--product', 'plutil',
+            '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH
+        ]
+        if self.args.verbose_build:
+            plutil_cmd.append('--verbose')
+
+        # On amazon-linux2 the gold linker (version 1.14) crashes when linking
+        # debug info for swift-foundation. Workaround this issue by building without
+        # debug info. In order to re-use the build products from swiftfoundationtests
+        # we need to build without debug info here as well. rdar://137760869
+        if host_target.startswith('linux'):
+            plutil_cmd += ['-Xswiftc', '-gnone']
+
+        shell.call(plutil_cmd, env={
+            'SWIFTCI_USE_LOCAL_DEPS': '1',
+            'DISPATCH_INCLUDE_PATH': include_path
+        })
+
         cmd = [
             swift_exec,
             'test',
             '--toolchain', self.install_toolchain_path(host_target),
             '--configuration', self.configuration(),
             '--scratch-path', self.build_dir,
-            '--package-path', package_path
+            '--package-path', package_path,
+            '--test-product', 'swift-corelibs-foundationPackageTests',
+            '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH
         ]
         if self.args.verbose_build:
             cmd.append('--verbose')
+
+        # On amazon-linux2 the gold linker (version 1.14) crashes when linking
+        # debug info for swift-foundation. Workaround this issue by building without
+        # debug info. In order to re-use the build products from swiftfoundationtests
+        # we need to build without debug info here as well. rdar://137760869
+        if host_target.startswith('linux'):
+            cmd += ['-Xswiftc', '-gnone']
+
         shell.call(cmd, env={
             'SWIFTCI_USE_LOCAL_DEPS': '1',
             'DISPATCH_INCLUDE_PATH': include_path

utils/swift_build_support/swift_build_support/products/foundationtests.py

@@ -42,8 +42,8 @@ def is_before_build_script_impl_product(cls):
         return False
 
     @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        return True
+    def swiftpm_unified_build_product_arena(cls):
+        return 'unified'
 
     def should_build(self, host_target):
         return True

utils/swift_build_support/swift_build_support/products/indexstoredb.py

@@ -89,13 +89,16 @@ def is_ignore_install_all_product(cls):
         return False
 
     @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        """is_swiftpm_unified_build_product -> bool
+    def swiftpm_unified_build_product_arena(cls):
+        """swiftpm_unified_build_product_arena -> Optional[str]
 
-        Whether this product should be built in the unified build of SwiftPM
-        products.
+        If this product should be built in the unified build of SwiftPM
+        products, a name that identifies the build directory for the unified
+        build. This should be 'unified' or 'foundationtests'. Foundation and
+        the rest can't share a unified build because of
+        https://github.com/swiftlang/swift-package-manager/issues/8344
         """
-        return False
+        return None
 
     @classmethod
     def is_nondarwin_only_build_product(cls):

utils/swift_build_support/swift_build_support/products/product.py

@@ -47,8 +47,8 @@ def is_before_build_script_impl_product(cls):
         return False
 
     @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        return True
+    def swiftpm_unified_build_product_arena(cls):
+        return 'unified'
 
     def package_name(self):
         return 'SourceKitStressTester'

utils/swift_build_support/swift_build_support/products/skstresstester.py

@@ -42,8 +42,8 @@ def is_before_build_script_impl_product(cls):
         return False
 
     @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        return True
+    def swiftpm_unified_build_product_arena(cls):
+        return 'unified'
 
     def should_build(self, host_target):
         return True

utils/swift_build_support/swift_build_support/products/sourcekitlsp.py

@@ -27,10 +27,6 @@ def is_build_script_impl_product(cls):
     def is_before_build_script_impl_product(cls):
         return False
 
-    @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        return False
-
     def should_build(self, host_target):
         return self.args.build_stdlib_docs
 

utils/swift_build_support/swift_build_support/products/stdlib_docs.py

@@ -46,8 +46,8 @@ def is_before_build_script_impl_product(cls):
         return False
 
     @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        return True
+    def swiftpm_unified_build_product_arena(cls):
+        return 'unified'
 
     def run_build_script_helper(self, action, host_target, additional_params=[]):
         script_path = os.path.join(

utils/swift_build_support/swift_build_support/products/swiftdocc.py

@@ -34,10 +34,6 @@ def is_build_script_impl_product(cls):
     def is_before_build_script_impl_product(cls):
         return False
 
-    @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        return False
-
     def should_build(self, host_target):
         # Swift-DocC-Render is a pre-built, installable artifact.
         return False

utils/swift_build_support/swift_build_support/products/swiftdoccrender.py

@@ -47,8 +47,8 @@ def is_before_build_script_impl_product(cls):
         return False
 
     @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        return True
+    def swiftpm_unified_build_product_arena(cls):
+        return 'unified'
 
     def configuration(self):
         return 'release' if self.is_release() else 'debug'

utils/swift_build_support/swift_build_support/products/swiftformat.py

@@ -12,6 +12,8 @@
 
 import os
 
+from build_swift.build_swift.constants import MULTIROOT_DATA_FILE_PATH
+
 from . import cmark
 from . import foundation
 from . import libcxx
@@ -35,6 +37,10 @@ def is_build_script_impl_product(cls):
     def is_before_build_script_impl_product(cls):
         return False
 
+    @classmethod
+    def swiftpm_unified_build_product_arena(cls):
+        return 'foundationtests'
+
     @classmethod
     def is_ignore_install_all_product(cls):
         return True
@@ -63,13 +69,20 @@ def test(self, host_target):
         )
         package_path = os.path.join(self.source_dir, '..', 'swift-foundation')
         package_path = os.path.abspath(package_path)
+        include_path = os.path.join(
+            self.install_toolchain_path(host_target),
+            'lib',
+            'swift'
+        )
         cmd = [
             swift_exec,
             'test',
             '--toolchain', self.install_toolchain_path(host_target),
             '--configuration', self.configuration(),
             '--scratch-path', self.build_dir,
-            '--package-path', package_path
+            '--package-path', package_path,
+            '--test-product', 'swift-foundationPackageTests',
+            '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH
         ]
         if self.args.verbose_build:
             cmd.append('--verbose')
@@ -80,7 +93,10 @@ def test(self, host_target):
         if host_target.startswith('linux'):
             cmd += ['-Xswiftc', '-gnone']
 
-        shell.call(cmd, env={'SWIFTCI_USE_LOCAL_DEPS': '1'})
+        shell.call(cmd, env={
+            'SWIFTCI_USE_LOCAL_DEPS': '1',
+            'DISPATCH_INCLUDE_PATH': include_path
+        })
 
     @classmethod
     def get_dependencies(cls):

utils/swift_build_support/swift_build_support/products/swiftfoundationtests.py

@@ -46,8 +46,8 @@ def is_before_build_script_impl_product(cls):
         return False
 
     @classmethod
-    def is_swiftpm_unified_build_product(cls):
-        return True
+    def swiftpm_unified_build_product_arena(cls):
+        return 'unified'
 
     def run_swiftsyntax_build_script(self, target, command, additional_params=[]):
         script_path = os.path.join(self.source_dir, 'SwiftSyntaxDevUtils')

utils/swift_build_support/swift_build_support/products/swiftsyntax.py

@@ -29,14 +29,13 @@ def build_dir(self, deployment_target, product):
         return os.path.join(self.build_root,
                             '%s-%s' % (product, deployment_target))
 
-    def swiftpm_unified_build_dir(self, deployment_target):
+    def swiftpm_unified_build_dir(self, arena, deployment_target):
         """ swiftpm_unified_build_dir() -> str
 
         Build directory that all SwiftPM unified build products share.
         """
         return os.path.join(self.build_root,
-                            'unified-swiftpm-build-%s' %
-                            deployment_target)
+                            f'{arena}-swiftpm-build-{deployment_target}')
 
 
 def compute_build_subdir(args):