Extend embuilder deferred building mode to ports

.circleci/config.yml

@@ -499,7 +499,7 @@ jobs:
       - build-libs
       - run:
           name: Clean build directory
-          command: rm -rf ~/cache/build
+          command: rm -rf ~/cache/build ~/cache/ports-builds
       - persist
   # Perhaps we don't need to run this suite with every commit. Consider moving this to FYI bot.
   test-posixtest:

embuilder.py

@@ -170,8 +170,8 @@ def clear_port(port_name):
 
 
 def build_port(port_name):
-  with get_port_variant(port_name) as port_name:
-    ports.build_port(port_name, settings)
+  with get_port_variant(port_name) as port_name_base:
+    ports.build_port(port_name_base, settings, deferred=USE_NINJA)
 
 
 def get_system_tasks():
@@ -221,6 +221,7 @@ def main():
   # a system library into the cache, causing trouble.
   cache.setup()
   shared.check_sanity()
+  utils.safe_ensure_dirs(cache.get_path('build'))
 
   if args.lto:
     settings.LTO = args.lto

tools/cache.py

@@ -177,8 +177,6 @@ def get(shortname, creator, what=None, force=False, quiet=False, deferred=False)
     logger.info(message)
     utils.safe_ensure_dirs(cachename.parent)
     creator(str(cachename))
-    if not deferred:
-      assert cachename.exists()
     if not quiet:
       logger.info(' - ok')
 

tools/ports/__init__.py

@@ -6,6 +6,7 @@
 import logging
 import hashlib
 import os
+from pathlib import Path
 import shutil
 import glob
 import importlib.util
@@ -36,6 +37,8 @@
 
 logger = logging.getLogger('ports')
 
+build_deferred = False
+
 
 def init_port(name, port):
   ports.append(port)
@@ -176,7 +179,9 @@ def install_headers(src_dir, pattern='*.h', target=None):
 
   @staticmethod
   def build_port(src_dir, output_path, port_name, includes=[], flags=[], cxxflags=[], exclude_files=[], exclude_dirs=[], srcs=[]):  # noqa
-    build_dir = os.path.join(Ports.get_build_dir(), port_name)
+    mangled_name = str(Path(output_path).relative_to(cache.get_sysroot(True))).replace(os.sep, '_')
+    build_dir = os.path.join(Ports.get_build_dir(), mangled_name)
+    logger.debug(f'build_port: {port_name} {output_path} in {build_dir}')
     if srcs:
       srcs = [os.path.join(src_dir, s) for s in srcs]
     else:
@@ -199,7 +204,8 @@ def build_port(src_dir, output_path, port_name, includes=[], flags=[], cxxflags=
       ninja_file = os.path.join(build_dir, 'build.ninja')
       system_libs.ensure_sysroot()
       system_libs.create_ninja_file(srcs, ninja_file, output_path, cflags=cflags)
-      system_libs.run_ninja(build_dir)
+      if not build_deferred:
+        system_libs.run_ninja(build_dir)
     else:
       commands = []
       objects = []
@@ -525,7 +531,10 @@ def get_needed_ports(settings):
   return needed
 
 
-def build_port(port_name, settings):
+def build_port(port_name, settings, deferred=False):
+  global build_deferred
+  build_deferred = deferred
+  logger.debug(f'build_port {port_name}, {"deferred" if deferred else ""}')
   port = ports_by_name[port_name]
   port_set = OrderedSet([port])
   resolve_dependencies(port_set, settings)
@@ -573,10 +582,13 @@ def add_cflags(args, settings): # noqa: U100
 
   # Now get (i.e. build) the ports in dependency order.  This is important because the
   # headers from one ports might be needed before we can build the next.
+  global build_deferred
+  assert not build_deferred, "add_cflags shouldn't be called from embuilder"
+  build_deferred = True
   for port in dependency_order(needed):
     port.get(Ports, settings, shared)
     args += port.process_args(Ports)
-
+  build_deferred = False
 
 def show_ports():
   sorted_ports = sorted(ports, key=lambda p: p.name)

tools/ports/sdl2_gfx.py

@@ -16,8 +16,6 @@ def needed(settings):
 
 
 def get(ports, settings, shared):
-  sdl_build = os.path.join(ports.get_build_dir(), 'sdl2')
-  assert os.path.exists(sdl_build), 'You must use SDL2 to use SDL2_gfx'
   ports.fetch_project('sdl2_gfx', f'https://github.com/svn2github/sdl2_gfx/archive/{TAG}.zip', sha512hash=HASH)
 
   def create(final):

tools/ports/sdl2_image.py

@@ -52,8 +52,6 @@ def get_lib_name(settings):
 
 
 def get(ports, settings, shared):
-  sdl_build = os.path.join(ports.get_build_dir(), 'sdl2')
-  assert os.path.exists(sdl_build), 'You must use SDL2 to use SDL2_image'
   ports.fetch_project('sdl2_image', f'https://github.com/libsdl-org/SDL_image/archive/refs/tags/{TAG}.zip', sha512hash=HASH)
   libname = get_lib_name(settings)
 

tools/ports/sdl2_mixer.py

@@ -36,8 +36,6 @@ def get_lib_name(settings):
 
 
 def get(ports, settings, shared):
-  sdl_build = os.path.join(ports.get_build_dir(), 'sdl2')
-  assert os.path.exists(sdl_build), 'You must use SDL2 to use SDL2_mixer'
   ports.fetch_project('sdl2_mixer', f'https://github.com/libsdl-org/SDL_mixer/archive/{TAG}.zip', sha512hash=HASH)
   libname = get_lib_name(settings)
 

tools/ports/sdl2_net.py

@@ -16,8 +16,6 @@ def needed(settings):
 
 
 def get(ports, settings, shared):
-  sdl_build = os.path.join(ports.get_build_dir(), 'sdl2')
-  assert os.path.exists(sdl_build), 'You must use SDL2 to use SDL2_net'
   ports.fetch_project('sdl2_net', f'https://github.com/emscripten-ports/SDL2_net/archive/{TAG}.zip', sha512hash=HASH)
 
   def create(final):