From 952d572218308de2767cf11f4e64cc632616cfad Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 27 Mar 2023 16:10:45 +0200 Subject: [PATCH] Allow plugging in observer --- patch.sh | 2 +- ...rface-compilation-fallback-on-stdlib.patch | 29 +++++ ...05-add-finished-to-frontend-observer.patch | 49 ++++++++ .../swift/06-remove-tests-from-cmake.patch | 23 ++++ ...ntroduce-pluggable-frontend-observer.patch | 118 ++++++++++++++++++ .../08-avoid-opaque-identifier-pointers.patch | 107 ++++++++++++++++ pkg_swift_llvm.py | 87 +++---------- 7 files changed, 345 insertions(+), 70 deletions(-) create mode 100644 patches/swift/04-turn-on-interface-compilation-fallback-on-stdlib.patch create mode 100644 patches/swift/05-add-finished-to-frontend-observer.patch create mode 100644 patches/swift/06-remove-tests-from-cmake.patch create mode 100644 patches/swift/07-introduce-pluggable-frontend-observer.patch create mode 100644 patches/swift/08-avoid-opaque-identifier-pointers.patch diff --git a/patch.sh b/patch.sh index c61e6df..513e26f 100755 --- a/patch.sh +++ b/patch.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e PATCH_DIR="$(cd "$(dirname "$0")/patches"; pwd)" diff --git a/patches/swift/04-turn-on-interface-compilation-fallback-on-stdlib.patch b/patches/swift/04-turn-on-interface-compilation-fallback-on-stdlib.patch new file mode 100644 index 0000000..c63e5a3 --- /dev/null +++ b/patches/swift/04-turn-on-interface-compilation-fallback-on-stdlib.patch @@ -0,0 +1,29 @@ +Remove block of code that is turning off auto-compilation of stdlib `.swiftinterface` +files. + +diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp +index 2a490a680d8..8d8f48aeed0 100644 +--- a/lib/Frontend/ModuleInterfaceLoader.cpp ++++ b/lib/Frontend/ModuleInterfaceLoader.cpp +@@ -726,21 +726,6 @@ class ModuleInterfaceLoaderImpl { + << "; deferring to serialized module loader\n"); + UsableModulePath = adjacentMod; + return std::make_error_code(std::errc::not_supported); +- } else if (isInResourceDir(adjacentMod) && +- loadMode == ModuleLoadingMode::PreferSerialized) { +- // Special-case here: If we're loading a .swiftmodule from the resource +- // dir adjacent to the compiler, defer to the serialized loader instead +- // of falling back. This is mainly to support development of Swift, +- // where one might change the module format version but forget to +- // recompile the standard library. If that happens, don't fall back +- // and silently recompile the standard library -- instead, error like +- // we used to. +- LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module in the " +- "resource-dir at " +- << adjacentMod +- << "; deferring to serialized module loader " +- "to diagnose\n"); +- return std::make_error_code(std::errc::not_supported); + } else { + LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module at " + << adjacentMod << "\n"); diff --git a/patches/swift/05-add-finished-to-frontend-observer.patch b/patches/swift/05-add-finished-to-frontend-observer.patch new file mode 100644 index 0000000..1e39234 --- /dev/null +++ b/patches/swift/05-add-finished-to-frontend-observer.patch @@ -0,0 +1,49 @@ +Add a `finished(int status)` method to `FrontendObserver`, fired when the frontend has finished. + +diff --git a/include/swift/FrontendTool/FrontendTool.h b/include/swift/FrontendTool/FrontendTool.h +index 184e6196918..ef5c3eafe69 100644 +--- a/include/swift/FrontendTool/FrontendTool.h ++++ b/include/swift/FrontendTool/FrontendTool.h +@@ -53,6 +53,9 @@ public: + /// The frontend has executed the SIL optimization and diagnostics pipelines. + virtual void performedSILProcessing(SILModule &module); + ++ /// The frontend has finished executing with the given return value ++ virtual void finished(int status); ++ + // TODO: maybe enhance this interface to hear about IRGen and LLVM + // progress. + }; +diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp +index 811fb912f8a..afa2034aa71 100644 +--- a/lib/FrontendTool/FrontendTool.cpp ++++ b/lib/FrontendTool/FrontendTool.cpp +@@ -1924,7 +1924,7 @@ public: + }; + }; + +-int swift::performFrontend(ArrayRef Args, ++static int performFrontendImpl(ArrayRef Args, + const char *Argv0, void *MainAddr, + FrontendObserver *observer) { + INITIALIZE_LLVM(); +@@ -2263,8 +2263,19 @@ int swift::performFrontend(ArrayRef Args, + return r; + } + ++int swift::performFrontend(ArrayRef Args, ++ const char *Argv0, void *MainAddr, ++ FrontendObserver *observer) { ++ auto ret = performFrontendImpl(Args, Argv0, MainAddr, observer); ++ if (observer) { ++ observer->finished(ret); ++ } ++ return ret; ++} ++ + void FrontendObserver::parsedArgs(CompilerInvocation &invocation) {} + void FrontendObserver::configuredCompiler(CompilerInstance &instance) {} + void FrontendObserver::performedSemanticAnalysis(CompilerInstance &instance) {} + void FrontendObserver::performedSILGeneration(SILModule &module) {} + void FrontendObserver::performedSILProcessing(SILModule &module) {} ++void FrontendObserver::finished(int status) {} diff --git a/patches/swift/06-remove-tests-from-cmake.patch b/patches/swift/06-remove-tests-from-cmake.patch new file mode 100644 index 0000000..f590dfa --- /dev/null +++ b/patches/swift/06-remove-tests-from-cmake.patch @@ -0,0 +1,23 @@ +diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt +index 60b0ae22368..f99551bbf7b 100644 +--- a/tools/CMakeLists.txt ++++ b/tools/CMakeLists.txt +@@ -40,18 +40,6 @@ if(SWIFT_BUILD_SYNTAXPARSERLIB) + endif() + endif() + +-if(SWIFT_INCLUDE_TESTS OR SWIFT_INCLUDE_TEST_BINARIES) +- add_swift_tool_subdirectory(swift-ide-test) +- add_swift_tool_subdirectory(swift-remoteast-test) +- add_swift_tool_subdirectory(lldb-moduleimport-test) +- add_swift_tool_subdirectory(swift-syntax-test) +-endif() +- +-if(LLVM_USE_SANITIZE_COVERAGE) +-add_swift_tool_subdirectory(swift-demangle-fuzzer) +-add_swift_tool_subdirectory(swift-reflection-fuzzer) +-endif() +- + if(SWIFT_BUILD_SOURCEKIT) + add_swift_tool_subdirectory(SourceKit) + endif() diff --git a/patches/swift/07-introduce-pluggable-frontend-observer.patch b/patches/swift/07-introduce-pluggable-frontend-observer.patch new file mode 100644 index 0000000..5ad544e --- /dev/null +++ b/patches/swift/07-introduce-pluggable-frontend-observer.patch @@ -0,0 +1,118 @@ +Make calls to `performFrontend` use an observer taken from `getFrontendObserver`, which +is provided by a shared library. Substituting the shared library can plug in our +extractor. + +diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake +index 6d93e52f1d5..10225f40fb1 100644 +--- a/cmake/modules/AddSwift.cmake ++++ b/cmake/modules/AddSwift.cmake +@@ -715,6 +715,7 @@ function(add_swift_host_tool executable) + + # Include the abi stable system stdlib in our rpath. + list(APPEND RPATH_LIST "/usr/lib/swift") ++ list(APPEND RPATH_LIST "@executable_path/../lib") + + elseif(BOOTSTRAPPING_MODE STREQUAL "CROSSCOMPILE-WITH-HOSTLIBS") + +@@ -798,7 +799,7 @@ function(add_swift_host_tool executable) + if(BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS") + set_target_properties(${executable} PROPERTIES + BUILD_WITH_INSTALL_RPATH YES +- INSTALL_RPATH "${host_lib_dir}") ++ INSTALL_RPATH "${host_lib_dir};$ORIGIN/../lib") + else() + set_target_properties(${executable} PROPERTIES + BUILD_WITH_INSTALL_RPATH YES +diff --git a/include/swift/DriverTool/FrontendObserver.h b/include/swift/DriverTool/FrontendObserver.h +new file mode 100644 +index 00000000000..4ac9b299a13 +--- /dev/null ++++ b/include/swift/DriverTool/FrontendObserver.h +@@ -0,0 +1,10 @@ ++#pragma once ++ ++#include "llvm/ADT/ArrayRef.h" ++#include "swift/FrontendTool/FrontendTool.h" ++ ++namespace swift { ++ ++FrontendObserver* getFrontendObserver(llvm::ArrayRef argv); ++ ++} // namespace swift +diff --git a/lib/DriverTool/CMakeLists.txt b/lib/DriverTool/CMakeLists.txt +index 869c00fece9..b1f49a976d5 100644 +--- a/lib/DriverTool/CMakeLists.txt ++++ b/lib/DriverTool/CMakeLists.txt +@@ -14,16 +14,24 @@ set(driver_common_libs + swiftSymbolGraphGen + LLVMBitstreamReader) + ++add_swift_host_library(swiftFrontendObserver SHARED ++ swift_frontend_observer.cpp) ++target_link_libraries(swiftFrontendObserver ++ PUBLIC ++ swiftFrontendTool) ++ + add_swift_host_library(swiftDriverTool STATIC + ${driver_sources_and_options} + ) + target_link_libraries(swiftDriverTool + PUBLIC +- ${driver_common_libs}) ++ ${driver_common_libs} ++ swiftFrontendObserver) + + # If building as part of clang, make sure the headers are installed. + if(NOT SWIFT_BUILT_STANDALONE) + add_dependencies(swiftDriverTool clang-resource-headers) + endif() + ++set_swift_llvm_is_available(swiftFrontendObserver) + set_swift_llvm_is_available(swiftDriverTool) +diff --git a/lib/DriverTool/driver.cpp b/lib/DriverTool/driver.cpp +index f71e2de9eae..a500e30827f 100644 +--- a/lib/DriverTool/driver.cpp ++++ b/lib/DriverTool/driver.cpp +@@ -31,6 +31,7 @@ + #include "swift/Frontend/PrintingDiagnosticConsumer.h" + #include "swift/FrontendTool/FrontendTool.h" + #include "swift/DriverTool/DriverTool.h" ++#include "swift/DriverTool/FrontendObserver.h" + #include "llvm/ADT/SmallVector.h" + #include "llvm/Support/CommandLine.h" + #include "llvm/Support/ConvertUTF.h" +@@ -197,7 +198,8 @@ static int run_driver(StringRef ExecName, + if (FirstArg == "-frontend") { + return performFrontend(llvm::makeArrayRef(argv.data()+2, + argv.data()+argv.size()), +- argv[0], (void *)(intptr_t)getExecutablePath); ++ argv[0], (void *)(intptr_t)getExecutablePath, ++ getFrontendObserver(argv)); + } + if (FirstArg == "-modulewrap") { + return modulewrap_main(llvm::makeArrayRef(argv.data()+2, +@@ -211,7 +213,8 @@ static int run_driver(StringRef ExecName, + && ExecName == "swift-frontend") { + return performFrontend(llvm::makeArrayRef(argv.data()+1, + argv.data()+argv.size()), +- argv[0], (void *)(intptr_t)getExecutablePath); ++ argv[0], (void *)(intptr_t)getExecutablePath, ++ getFrontendObserver(argv)); + } + + if (FirstArg == "repl") { +diff --git a/lib/DriverTool/swift_frontend_observer.cpp b/lib/DriverTool/swift_frontend_observer.cpp +new file mode 100644 +index 00000000000..e16b2f970cd +--- /dev/null ++++ b/lib/DriverTool/swift_frontend_observer.cpp +@@ -0,0 +1,9 @@ ++#include "swift/DriverTool/FrontendObserver.h" ++ ++namespace swift { ++ ++FrontendObserver* getFrontendObserver(llvm::ArrayRef) { ++ return nullptr; ++} ++ ++} // namespace swift diff --git a/patches/swift/08-avoid-opaque-identifier-pointers.patch b/patches/swift/08-avoid-opaque-identifier-pointers.patch new file mode 100644 index 0000000..2c9d735 --- /dev/null +++ b/patches/swift/08-avoid-opaque-identifier-pointers.patch @@ -0,0 +1,107 @@ +When using our extractor as a plugin, it turns out there special opaque pointers +are different in the frontend and our extractor contexts, leading to special +members (subscripts, inits, deinits) to lose their name. + +This avoids using this opaque data pointers and relies instead on actual data +contained therein. This special data is distinguished from normal identifiers by +using the unprintable `\1` as first character, and `s`, `c` or `d` as second +character for subscript, constructors and destructors respectively. + +diff --git a/include/swift/AST/Identifier.h b/include/swift/AST/Identifier.h +index 6f65385e097..6ec0da0dd09 100644 +--- a/include/swift/AST/Identifier.h ++++ b/include/swift/AST/Identifier.h +@@ -72,8 +72,12 @@ private: + && "Identifier pointer does not use any spare bits"); + } + +- /// A type with the alignment expected of a valid \c Identifier::Pointer . +- struct alignas(uint64_t) Aligner {}; ++ /** A type with the alignment expected of a valid \c Identifier::Pointer ++ * The actual value is used to distinguish special identifiers ++ */ ++ struct alignas(uint64_t) Aligner { ++ char value[3]; ++ }; + + static_assert(alignof(Aligner) >= RequiredAlignment, + "Identifier table will provide enough spare bits"); +@@ -271,7 +275,7 @@ public: + }; + + private: +- /// In a special DeclName representing a subscript, this opaque pointer ++ /// In a special DeclName representing a subscript, this opaque structure + /// is used as the data of the base name identifier. + /// This is an implementation detail that should never leak outside of + /// DeclName. +@@ -289,29 +293,19 @@ public: + DeclBaseName(Identifier I) : Ident(I) {} + + static DeclBaseName createSubscript() { +- return DeclBaseName(Identifier((const char *)&SubscriptIdentifierData)); ++ return DeclBaseName(Identifier(SubscriptIdentifierData.value)); + } + + static DeclBaseName createConstructor() { +- return DeclBaseName(Identifier((const char *)&ConstructorIdentifierData)); ++ return DeclBaseName(Identifier(ConstructorIdentifierData.value)); + } + + static DeclBaseName createDestructor() { +- return DeclBaseName(Identifier((const char *)&DestructorIdentifierData)); +- } +- +- Kind getKind() const { +- if (Ident.get() == (const char *)&SubscriptIdentifierData) { +- return Kind::Subscript; +- } else if (Ident.get() == (const char *)&ConstructorIdentifierData) { +- return Kind::Constructor; +- } else if (Ident.get() == (const char *)&DestructorIdentifierData) { +- return Kind::Destructor; +- } else { +- return Kind::Normal; +- } ++ return DeclBaseName(Identifier(DestructorIdentifierData.value)); + } + ++ Kind getKind() const; ++ + bool isSpecial() const { return getKind() != Kind::Normal; } + + bool isSubscript() const { return getKind() == Kind::Subscript; } +diff --git a/lib/AST/Identifier.cpp b/lib/AST/Identifier.cpp +index da9ffbc3ee9..d870d0d85e6 100644 +--- a/lib/AST/Identifier.cpp ++++ b/lib/AST/Identifier.cpp +@@ -23,9 +23,21 @@ + #include "clang/Basic/CharInfo.h" + using namespace swift; + +-constexpr const Identifier::Aligner DeclBaseName::SubscriptIdentifierData{}; +-constexpr const Identifier::Aligner DeclBaseName::ConstructorIdentifierData{}; +-constexpr const Identifier::Aligner DeclBaseName::DestructorIdentifierData{}; ++constexpr const Identifier::Aligner DeclBaseName::SubscriptIdentifierData{"\1s"}; ++constexpr const Identifier::Aligner DeclBaseName::ConstructorIdentifierData{"\1c"}; ++constexpr const Identifier::Aligner DeclBaseName::DestructorIdentifierData{"\1d"}; ++ ++DeclBaseName::Kind DeclBaseName::getKind() const { ++ if (!Ident.get() || Ident.get()[0] != '\1') { ++ return Kind::Normal; ++ } ++ switch(Ident.get()[1]) { ++ case 's': return Kind::Subscript; ++ case 'c': return Kind::Constructor; ++ case 'd': return Kind::Destructor; ++ default: return Kind::Normal; ++ } ++} + + raw_ostream &llvm::operator<<(raw_ostream &OS, Identifier I) { + if (I.get() == nullptr) +@@ -305,5 +317,3 @@ StringRef ObjCSelector::getString(llvm::SmallVectorImpl &scratch) const { + void ObjCSelector::dump() const { + llvm::errs() << *this << "\n"; + } +- +- diff --git a/pkg_swift_llvm.py b/pkg_swift_llvm.py index 345857d..db698da 100755 --- a/pkg_swift_llvm.py +++ b/pkg_swift_llvm.py @@ -34,7 +34,7 @@ def getoptions(): return opts -Libs = namedtuple("Libs", ("archive", "static", "shared", "linker_flags")) +Libs = namedtuple("Libs", ("static", "shared", "linker_flags")) EXPORTED_LIB = "CodeQLSwiftFrontendTool" @@ -74,11 +74,11 @@ def get_libs(configured): print("extracting linking information from dummy project") with open(configured / "CMakeFiles" / "codeql-swift-artifacts.dir" / "link.txt") as link: libs = link.read().split() - libs = libs[libs.index('codeql-swift-artifacts')+1:] # skip up to -o dummy - ret = Libs([], [], [], []) + libs = libs[libs.index('codeql-swift-artifacts') + 1:] # skip up to -o dummy + ret = Libs([], [], []) for l in libs: if l.endswith(".a"): - ret.static.append(str((configured / l).resolve())) + ret.static.append((configured / l).resolve()) elif l.endswith(".so") or l.endswith(".tbd") or l.endswith(".dylib"): l = pathlib.Path(l).stem ret.shared.append(f"-l{l[3:]}") # drop 'lib' prefix and '.so' suffix @@ -97,55 +97,6 @@ def get_tgt(tgt, filename): return tgt.resolve() -def create_static_lib(tgt, libs): - tgt = get_tgt(tgt, f"lib{EXPORTED_LIB}.a") - print(f"packaging {tgt.name}") - if sys.platform == 'linux': - includedlibs = "\n".join(f"addlib {l}" for l in libs.archive + libs.static) - mriscript = f"create {tgt}\n{includedlibs}\nsave\nend" - run(["ar", "-M"], cwd=tgt.parent, input=mriscript) - else: - libtool_args = ["libtool", "-static"] - libtool_args.extend(libs.archive) - libtool_args.extend(libs.static) - libtool_args.append("-o") - libtool_args.append(str(tgt)) - run(libtool_args, cwd=tgt.parent) - return tgt - - -def create_shared_lib(tgt, libs): - ext = "so" - if sys.platform != 'linux': - ext = "dylib" - libname = f"lib{EXPORTED_LIB}.{ext}" - tgt = get_tgt(tgt, libname) - print(f"packaging {libname}") - compiler = os.environ.get("CC", "clang") - cmd = [compiler, "-shared"] - cmd.extend(libs.linker_flags) - - if sys.platform == 'linux': - cmd.append("-Wl,--whole-archive") - else: - cmd.append("-Wl,-all_load") - - cmd.append(f"-o{tgt}") - cmd.extend(libs.archive) - - if sys.platform == 'linux': - cmd.append("-Wl,--no-whole-archive") - else: - cmd.append("-lc++") - - cmd.extend(libs.static) - cmd.extend(libs.shared) - run(cmd, cwd=tgt.parent) - if sys.platform != "linux": - run(["install_name_tool", "-id", f"@executable_path/{libname}", libname], cwd=tgt.parent) - return tgt - - def copy_includes(src, tgt): print("copying includes") for dir, exts in (("include", ("h", "def", "inc")), ("stdlib", ("h",))): @@ -159,7 +110,7 @@ def copy_includes(src, tgt): def export_sdk(tgt, swift_source_tree, swift_build_tree): print("assembling sdk") - srcdir = swift_build_tree/ "lib" / "swift" + srcdir = swift_build_tree / "lib" / "swift" tgtdir = tgt / "usr" / "lib" / "swift" if get_platform() == "linux": srcdir /= "linux" @@ -181,11 +132,11 @@ def export_stdlibs(exported_dir, swift_build_tree): ext = 'so' lib_dir = swift_build_tree / 'lib/swift' / platform stdlibs = [ - f'libswiftCore.{ext}', - 'libswiftCompatibility50.a', - 'libswiftCompatibility51.a', - 'libswiftCompatibilityConcurrency.a', - 'libswiftCompatibilityDynamicReplacements.a'] + f'libswiftCore.{ext}', + 'libswiftCompatibility50.a', + 'libswiftCompatibility51.a', + 'libswiftCompatibilityConcurrency.a', + 'libswiftCompatibilityDynamicReplacements.a'] for stdlib in stdlibs: lib_path = lib_dir / stdlib if lib_path.exists(): @@ -197,13 +148,8 @@ def export_stdlibs(exported_dir, swift_build_tree): def export_libs(exported_dir, libs, swift_build_tree): print("exporting libraries") - exportedlibs = [ - create_static_lib(exported_dir, libs), - create_shared_lib(exported_dir, libs) - ] - - for l in exportedlibs: - l.rename(exported_dir / l.name) + for i, l in enumerate(libs.static): + shutil.copy(l, exported_dir / f"lib{i:03}{l.name[3:]}") export_stdlibs(exported_dir, swift_build_tree) @@ -213,10 +159,14 @@ def export_headers(exported_dir, swift_source_tree, llvm_build_tree, swift_build llvm_source_tree = swift_source_tree.parent / 'llvm-project/llvm' clang_source_tree = swift_source_tree.parent / 'llvm-project/clang' clang_tools_build_tree = llvm_build_tree / 'tools/clang' - header_dirs = [ llvm_source_tree, clang_source_tree, swift_source_tree, llvm_build_tree, swift_build_tree, clang_tools_build_tree ] + header_dirs = [llvm_source_tree, clang_source_tree, swift_source_tree, llvm_build_tree, swift_build_tree, + clang_tools_build_tree] for h in header_dirs: copy_includes(h, exported_dir) +def export_frontend(exported_dir, swift_build_tree): + print("exporting swift-frontend") + shutil.copy(swift_build_tree / 'bin' / 'swift-frontend', exported_dir) def zip_dir(src, tgt): tgt = get_tgt(tgt, f"swift-prebuilt-{get_platform()}.zip") @@ -238,10 +188,9 @@ def main(opts): export_libs(exported, libs, opts.swift_build_tree) export_headers(exported, opts.swift_source_tree, opts.llvm_build_tree, opts.swift_build_tree) export_sdk(exported / "sdk", opts.swift_source_tree, opts.swift_build_tree) - + export_frontend(exported, opts.swift_build_tree) zip_dir(exported, opts.output) if __name__ == "__main__": main(getoptions()) -