Allow plugging in observer

patch.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 
 PATCH_DIR="$(cd "$(dirname "$0")/patches"; pwd)"
 

patches/swift/04-turn-on-interface-compilation-fallback-on-stdlib.patch

@@ -0,0 +1,29 @@
+Remove block of code that is turning off auto-compilation of stdlib `.swiftinterface`
+files.
+
+diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp
+index 2a490a680d8..8d8f48aeed0 100644
+--- a/lib/Frontend/ModuleInterfaceLoader.cpp
++++ b/lib/Frontend/ModuleInterfaceLoader.cpp
+@@ -726,21 +726,6 @@ class ModuleInterfaceLoaderImpl {
+                                   << "; deferring to serialized module loader\n");
+           UsableModulePath = adjacentMod;
+           return std::make_error_code(std::errc::not_supported);
+-        } else if (isInResourceDir(adjacentMod) &&
+-                   loadMode == ModuleLoadingMode::PreferSerialized) {
+-          // Special-case here: If we're loading a .swiftmodule from the resource
+-          // dir adjacent to the compiler, defer to the serialized loader instead
+-          // of falling back. This is mainly to support development of Swift,
+-          // where one might change the module format version but forget to
+-          // recompile the standard library. If that happens, don't fall back
+-          // and silently recompile the standard library -- instead, error like
+-          // we used to.
+-          LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module in the "
+-                                     "resource-dir at "
+-                                  << adjacentMod
+-                                  << "; deferring to serialized module loader "
+-                                     "to diagnose\n");
+-          return std::make_error_code(std::errc::not_supported);
+         } else {
+           LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module at "
+                                   << adjacentMod << "\n");

patches/swift/05-add-finished-to-frontend-observer.patch

@@ -0,0 +1,49 @@
+Add a `finished(int status)` method to `FrontendObserver`, fired when the frontend has finished.
+
+diff --git a/include/swift/FrontendTool/FrontendTool.h b/include/swift/FrontendTool/FrontendTool.h
+index 184e6196918..ef5c3eafe69 100644
+--- a/include/swift/FrontendTool/FrontendTool.h
++++ b/include/swift/FrontendTool/FrontendTool.h
+@@ -53,6 +53,9 @@ public:
+   /// The frontend has executed the SIL optimization and diagnostics pipelines.
+   virtual void performedSILProcessing(SILModule &module);
+
++  /// The frontend has finished executing with the given return value
++  virtual void finished(int status);
++
+   // TODO: maybe enhance this interface to hear about IRGen and LLVM
+   // progress.
+ };
+diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
+index 811fb912f8a..afa2034aa71 100644
+--- a/lib/FrontendTool/FrontendTool.cpp
++++ b/lib/FrontendTool/FrontendTool.cpp
+@@ -1924,7 +1924,7 @@ public:
+   };
+ };
+
+-int swift::performFrontend(ArrayRef<const char *> Args,
++static int performFrontendImpl(ArrayRef<const char *> Args,
+                            const char *Argv0, void *MainAddr,
+                            FrontendObserver *observer) {
+   INITIALIZE_LLVM();
+@@ -2263,8 +2263,19 @@ int swift::performFrontend(ArrayRef<const char *> Args,
+   return r;
+ }
+
++int swift::performFrontend(ArrayRef<const char *> Args,
++                           const char *Argv0, void *MainAddr,
++                           FrontendObserver *observer) {
++  auto ret = performFrontendImpl(Args, Argv0, MainAddr, observer);
++  if (observer) {
++    observer->finished(ret);
++  }
++  return ret;
++}
++
+ void FrontendObserver::parsedArgs(CompilerInvocation &invocation) {}
+ void FrontendObserver::configuredCompiler(CompilerInstance &instance) {}
+ void FrontendObserver::performedSemanticAnalysis(CompilerInstance &instance) {}
+ void FrontendObserver::performedSILGeneration(SILModule &module) {}
+ void FrontendObserver::performedSILProcessing(SILModule &module) {}
++void FrontendObserver::finished(int status) {}

patches/swift/06-remove-tests-from-cmake.patch

@@ -0,0 +1,23 @@
+diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
+index 60b0ae22368..f99551bbf7b 100644
+--- a/tools/CMakeLists.txt
++++ b/tools/CMakeLists.txt
+@@ -40,18 +40,6 @@ if(SWIFT_BUILD_SYNTAXPARSERLIB)
+   endif()
+ endif()
+
+-if(SWIFT_INCLUDE_TESTS OR SWIFT_INCLUDE_TEST_BINARIES)
+-  add_swift_tool_subdirectory(swift-ide-test)
+-  add_swift_tool_subdirectory(swift-remoteast-test)
+-  add_swift_tool_subdirectory(lldb-moduleimport-test)
+-  add_swift_tool_subdirectory(swift-syntax-test)
+-endif()
+-
+-if(LLVM_USE_SANITIZE_COVERAGE)
+-add_swift_tool_subdirectory(swift-demangle-fuzzer)
+-add_swift_tool_subdirectory(swift-reflection-fuzzer)
+-endif()
+-
+ if(SWIFT_BUILD_SOURCEKIT)
+   add_swift_tool_subdirectory(SourceKit)
+ endif()

patches/swift/07-introduce-pluggable-frontend-observer.patch

@@ -0,0 +1,118 @@
+Make calls to `performFrontend` use an observer taken from `getFrontendObserver`, which
+is provided by a shared library. Substituting the shared library can plug in our
+extractor.
+
+diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake
+index 6d93e52f1d5..10225f40fb1 100644
+--- a/cmake/modules/AddSwift.cmake
++++ b/cmake/modules/AddSwift.cmake
+@@ -715,6 +715,7 @@ function(add_swift_host_tool executable)
+
+         # Include the abi stable system stdlib in our rpath.
+         list(APPEND RPATH_LIST "/usr/lib/swift")
++        list(APPEND RPATH_LIST "@executable_path/../lib")
+
+       elseif(BOOTSTRAPPING_MODE STREQUAL "CROSSCOMPILE-WITH-HOSTLIBS")
+
+@@ -798,7 +799,7 @@ function(add_swift_host_tool executable)
+       if(BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS")
+         set_target_properties(${executable} PROPERTIES
+           BUILD_WITH_INSTALL_RPATH YES
+-          INSTALL_RPATH  "${host_lib_dir}")
++          INSTALL_RPATH  "${host_lib_dir};$ORIGIN/../lib")
+       else()
+         set_target_properties(${executable} PROPERTIES
+           BUILD_WITH_INSTALL_RPATH YES
+diff --git a/include/swift/DriverTool/FrontendObserver.h b/include/swift/DriverTool/FrontendObserver.h
+new file mode 100644
+index 00000000000..4ac9b299a13
+--- /dev/null
++++ b/include/swift/DriverTool/FrontendObserver.h
+@@ -0,0 +1,10 @@
++#pragma once
++
++#include "llvm/ADT/ArrayRef.h"
++#include "swift/FrontendTool/FrontendTool.h"
++
++namespace swift {
++
++FrontendObserver* getFrontendObserver(llvm::ArrayRef<const char*> argv);
++
++} // namespace swift
+diff --git a/lib/DriverTool/CMakeLists.txt b/lib/DriverTool/CMakeLists.txt
+index 869c00fece9..b1f49a976d5 100644
+--- a/lib/DriverTool/CMakeLists.txt
++++ b/lib/DriverTool/CMakeLists.txt
+@@ -14,16 +14,24 @@ set(driver_common_libs
+                 swiftSymbolGraphGen
+                 LLVMBitstreamReader)
+
++add_swift_host_library(swiftFrontendObserver SHARED
++   swift_frontend_observer.cpp)
++target_link_libraries(swiftFrontendObserver
++        PUBLIC
++        swiftFrontendTool)
++
+ add_swift_host_library(swiftDriverTool STATIC
+   ${driver_sources_and_options}
+ )
+ target_link_libraries(swiftDriverTool
+                       PUBLIC
+-                      ${driver_common_libs})
++                      ${driver_common_libs}
++                      swiftFrontendObserver)
+
+ # If building as part of clang, make sure the headers are installed.
+ if(NOT SWIFT_BUILT_STANDALONE)
+   add_dependencies(swiftDriverTool clang-resource-headers)
+ endif()
+
++set_swift_llvm_is_available(swiftFrontendObserver)
+ set_swift_llvm_is_available(swiftDriverTool)
+diff --git a/lib/DriverTool/driver.cpp b/lib/DriverTool/driver.cpp
+index f71e2de9eae..a500e30827f 100644
+--- a/lib/DriverTool/driver.cpp
++++ b/lib/DriverTool/driver.cpp
+@@ -31,6 +31,7 @@
+ #include "swift/Frontend/PrintingDiagnosticConsumer.h"
+ #include "swift/FrontendTool/FrontendTool.h"
+ #include "swift/DriverTool/DriverTool.h"
++#include "swift/DriverTool/FrontendObserver.h"
+ #include "llvm/ADT/SmallVector.h"
+ #include "llvm/Support/CommandLine.h"
+ #include "llvm/Support/ConvertUTF.h"
+@@ -197,7 +198,8 @@ static int run_driver(StringRef ExecName,
+     if (FirstArg == "-frontend") {
+       return performFrontend(llvm::makeArrayRef(argv.data()+2,
+                                                 argv.data()+argv.size()),
+-                             argv[0], (void *)(intptr_t)getExecutablePath);
++                             argv[0], (void *)(intptr_t)getExecutablePath,
++                             getFrontendObserver(argv));
+     }
+     if (FirstArg == "-modulewrap") {
+       return modulewrap_main(llvm::makeArrayRef(argv.data()+2,
+@@ -211,7 +213,8 @@ static int run_driver(StringRef ExecName,
+         && ExecName == "swift-frontend") {
+       return performFrontend(llvm::makeArrayRef(argv.data()+1,
+                                                 argv.data()+argv.size()),
+-                             argv[0], (void *)(intptr_t)getExecutablePath);
++                             argv[0], (void *)(intptr_t)getExecutablePath,
++                             getFrontendObserver(argv));
+     }
+
+     if (FirstArg == "repl") {
+diff --git a/lib/DriverTool/swift_frontend_observer.cpp b/lib/DriverTool/swift_frontend_observer.cpp
+new file mode 100644
+index 00000000000..e16b2f970cd
+--- /dev/null
++++ b/lib/DriverTool/swift_frontend_observer.cpp
+@@ -0,0 +1,9 @@
++#include "swift/DriverTool/FrontendObserver.h"
++
++namespace swift {
++
++FrontendObserver* getFrontendObserver(llvm::ArrayRef<const char*>) {
++  return nullptr;
++}
++
++} // namespace swift

patches/swift/08-avoid-opaque-identifier-pointers.patch

@@ -0,0 +1,107 @@
+When using our extractor as a plugin, it turns out there special opaque pointers
+are different in the frontend and our extractor contexts, leading to special
+members (subscripts, inits, deinits) to lose their name.
+
+This avoids using this opaque data pointers and relies instead on actual data
+contained therein. This special data is distinguished from normal identifiers by
+using the unprintable `\1` as first character, and `s`, `c` or `d` as second
+character for subscript, constructors and destructors respectively.
+
+diff --git a/include/swift/AST/Identifier.h b/include/swift/AST/Identifier.h
+index 6f65385e097..6ec0da0dd09 100644
+--- a/include/swift/AST/Identifier.h
++++ b/include/swift/AST/Identifier.h
+@@ -72,8 +72,12 @@ private:
+            && "Identifier pointer does not use any spare bits");
+   }
+
+-  /// A type with the alignment expected of a valid \c Identifier::Pointer .
+-  struct alignas(uint64_t) Aligner {};
++  /** A type with the alignment expected of a valid \c Identifier::Pointer
++   *  The actual value is used to distinguish special identifiers
++   */
++  struct alignas(uint64_t) Aligner {
++    char value[3];
++  };
+
+   static_assert(alignof(Aligner) >= RequiredAlignment,
+                 "Identifier table will provide enough spare bits");
+@@ -271,7 +275,7 @@ public:
+   };
+
+ private:
+-  /// In a special DeclName representing a subscript, this opaque pointer
++  /// In a special DeclName representing a subscript, this opaque structure
+   /// is used as the data of the base name identifier.
+   /// This is an implementation detail that should never leak outside of
+   /// DeclName.
+@@ -289,29 +293,19 @@ public:
+   DeclBaseName(Identifier I) : Ident(I) {}
+
+   static DeclBaseName createSubscript() {
+-    return DeclBaseName(Identifier((const char *)&SubscriptIdentifierData));
++    return DeclBaseName(Identifier(SubscriptIdentifierData.value));
+   }
+
+   static DeclBaseName createConstructor() {
+-    return DeclBaseName(Identifier((const char *)&ConstructorIdentifierData));
++    return DeclBaseName(Identifier(ConstructorIdentifierData.value));
+   }
+
+   static DeclBaseName createDestructor() {
+-    return DeclBaseName(Identifier((const char *)&DestructorIdentifierData));
+-  }
+-
+-  Kind getKind() const {
+-    if (Ident.get() == (const char *)&SubscriptIdentifierData) {
+-      return Kind::Subscript;
+-    } else if (Ident.get() == (const char *)&ConstructorIdentifierData) {
+-      return Kind::Constructor;
+-    } else if (Ident.get() == (const char *)&DestructorIdentifierData) {
+-        return Kind::Destructor;
+-    } else {
+-      return Kind::Normal;
+-    }
++    return DeclBaseName(Identifier(DestructorIdentifierData.value));
+   }
+
++  Kind getKind() const;
++
+   bool isSpecial() const { return getKind() != Kind::Normal; }
+
+   bool isSubscript() const { return getKind() == Kind::Subscript; }
+diff --git a/lib/AST/Identifier.cpp b/lib/AST/Identifier.cpp
+index da9ffbc3ee9..d870d0d85e6 100644
+--- a/lib/AST/Identifier.cpp
++++ b/lib/AST/Identifier.cpp
+@@ -23,9 +23,21 @@
+ #include "clang/Basic/CharInfo.h"
+ using namespace swift;
+
+-constexpr const Identifier::Aligner DeclBaseName::SubscriptIdentifierData{};
+-constexpr const Identifier::Aligner DeclBaseName::ConstructorIdentifierData{};
+-constexpr const Identifier::Aligner DeclBaseName::DestructorIdentifierData{};
++constexpr const Identifier::Aligner DeclBaseName::SubscriptIdentifierData{"\1s"};
++constexpr const Identifier::Aligner DeclBaseName::ConstructorIdentifierData{"\1c"};
++constexpr const Identifier::Aligner DeclBaseName::DestructorIdentifierData{"\1d"};
++
++DeclBaseName::Kind DeclBaseName::getKind() const {
++  if (!Ident.get() || Ident.get()[0] != '\1') {
++    return Kind::Normal;
++  }
++  switch(Ident.get()[1]) {
++    case 's': return Kind::Subscript;
++    case 'c': return Kind::Constructor;
++    case 'd': return Kind::Destructor;
++    default: return Kind::Normal;
++  }
++}
+
+ raw_ostream &llvm::operator<<(raw_ostream &OS, Identifier I) {
+   if (I.get() == nullptr)
+@@ -305,5 +317,3 @@ StringRef ObjCSelector::getString(llvm::SmallVectorImpl<char> &scratch) const {
+ void ObjCSelector::dump() const {
+   llvm::errs() << *this << "\n";
+ }
+-
+-