0013-preprocess-Add-subcommand-preprocess.patch

From: Oliver Reiche <oliver.reiche@gmail.com>
Date: Thu, 28 Dec 2023 15:53:45 +0100
Subject: [preprocess] Add subcommand 'preprocess'

---
 share/man/must-backend.5.md       | 22 +++++++++---
 share/man/must.1.md               |  2 +-
 src/buildtool/common/cli.hpp      | 16 +++++++++
 src/buildtool/main/cli.cpp        | 17 ++++++++++
 src/buildtool/main/cli.hpp        |  4 ++-
 src/buildtool/main/main.cpp       | 56 +++++++++++++++++++++++++++++--
 src/other_tools/just_mr/utils.hpp | 11 +++++-
 7 files changed, 118 insertions(+), 10 deletions(-)

diff --git a/share/man/must-backend.5.md b/share/man/must-backend.5.md
--- a/share/man/must-backend.5.md
+++ b/share/man/must-backend.5.md
@@ -8,6 +8,7 @@ must backend - Backend for **`must`**(1), handling build-related subcommands.
 SYNOPSIS
 ========
 
+**`must`** **`preprocess`** \[*`OPTION`*\]... *`FILE`*  
 **`must`** {**`analyse`**|**`build`**} \[*`OPTION`*\]... \[\[*`module`*\] *`target`*\]  
 **`must`** **`install`** \[*`OPTION`*\]... **`-o`** *`OUTPUT_DIR`* \[\[*`module`*\] *`target`*\]  
 **`must`** **`install-cas`** \[*`OPTION`*\]... *`OBJECT_ID`*  
@@ -103,6 +104,12 @@ the multi-repository configuration file.
 SUBCOMMANDS
 ===========
 
+**`preprocess`**
+---------------------------------------
+
+Preprocess a Jsonnet file. It outputs a string on stdout in JSON syntax that
+might contain Justbuild expressions.
+
 **`analyse`**|**`build`**|**`install`**
 ---------------------------------------
 
@@ -430,29 +437,29 @@ Path to local log file. **`must`** will store the information printed on
 stderr in the log file along with the thread id and timestamp when the
 output has been generated.  
 Supported by:
-add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
+add-to-cas|preprocess|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
 
 **`--log-limit`** *`NUM`*  
 Log limit (higher is more verbose) in interval \[0,6\] (Default: 3).  
 Supported by:
-add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
+add-to-cas|preprocess|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
 
 **`--restrict-stderr-log-limit`** *`NUM`*  
 Restrict logging on console to the minimum of the specified **`--log-limit`**
 and the value specified in this option. The default is to not additionally
 restrict the log level at the console.  
 Supported by:
-add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
+add-to-cas|preprocess|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
 
 **`--plain-log`**  
 Do not use ANSI escape sequences to highlight messages.  
 Supported by:
-add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
+add-to-cas|preprocess|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
 
 **`--log-append`**  
 Append messages to log file instead of overwriting existing.  
 Supported by:
-add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
+add-to-cas|preprocess|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute.
 
 **`--expression-log-limit`** *`NUM`*  
 In error messages, truncate the entries in the enumeration of the active
@@ -533,6 +540,11 @@ Ensure that all installed artifacts are available in local CAS as well,
 even when using remote execution.  
 Supported by: install|traverse|install-cas.
 
+**`--root`** *`PATH`*  
+The file root used for resolving files to preprocess. Default: first parent of
+`FILE` that contains a workspace root marker or cwd if reading from stdin.  
+Supported by: preprocess.
+
 Parallelism options
 -------------------
 
diff --git a/share/man/must.1.md b/share/man/must.1.md
--- a/share/man/must.1.md
+++ b/share/man/must.1.md
@@ -12,7 +12,7 @@ SYNOPSIS
 **`must`** \[*`OPTION`*\]... {**`setup`**|**`setup-env`**} \[**`--all`**\] \[*`main-repo`*\]  
 **`must`** \[*`OPTION`*\]... **`fetch`** \[**`--all`**\] \[**`--backup-to-remote`**] \[**`-o`** *`fetch-dir`*\] \[*`main-repo`*\]  
 **`must`** \[*`OPTION`*\]... **`update`** \[*`repo`*\]...  
-**`must`** \[*`OPTION`*\]... {**`describe`**|**`analyse`**|**`build`**|**`install`**|**`install-cas`**|**`add-to-cas`**|**`rebuild`**|**`gc`**|**`traverse`**} \[*`SUBCMD_ARG`*\]...  
+**`must`** \[*`OPTION`*\]... {**`preprocess`**|**`describe`**|**`analyse`**|**`build`**|**`install`**|**`install-cas`**|**`add-to-cas`**|**`rebuild`**|**`gc`**|**`traverse`**} \[*`SUBCMD_ARG`*\]...  
 
 DESCRIPTION
 ===========
diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp
--- a/src/buildtool/common/cli.hpp
+++ b/src/buildtool/common/cli.hpp
@@ -199,6 +199,11 @@ struct ProtocolArguments final {
     HashFunction::Type hash_type = HashFunction::Type::GitSHA1;
 };
 
+struct PreprocessArguments {
+    std::filesystem::path path{};
+    std::optional<std::filesystem::path> root{};
+};
+
 static inline auto SetupCommonArguments(
     gsl::not_null<CLI::App*> const& app,
     gsl::not_null<CommonArguments*> const& clargs) {
@@ -801,6 +806,17 @@ static inline auto SetupServeArguments(
         ->required();
 }
 
+static inline auto SetupPreprocessArguments(
+    gsl::not_null<CLI::App*> const& app,
+    gsl::not_null<PreprocessArguments*> const& clargs) {
+    app->add_option("file", clargs->path, "File path (use - for stdin)")
+        ->required();
+    app->add_option(
+        "--root",
+        clargs->root,
+        "File root (Default: workspace root of file or cwd if stdin)");
+}
+
 static inline void SetupGcArguments(gsl::not_null<CLI::App*> const& app,
                                     gsl::not_null<GcArguments*> const& args) {
     app->add_flag("--no-rotate",
diff --git a/src/buildtool/main/cli.cpp b/src/buildtool/main/cli.cpp
--- a/src/buildtool/main/cli.cpp
+++ b/src/buildtool/main/cli.cpp
@@ -170,6 +170,15 @@ auto SetupServeServiceCommandArguments(
     SetupServeArguments(app, &clargs->serve);
 }
 
+/// \brief Setup arguments for sub command "just preprocess".
+auto SetupPreProcessCommandArguments(
+    gsl::not_null<CLI::App*> const& app,
+    gsl::not_null<CommandLineArguments*> const& clargs) {
+    // all other arguments will be read from config file
+    SetupLogArguments(app, &clargs->log);
+    SetupPreprocessArguments(app, &clargs->preprocess);
+}
+
 }  // namespace
 
 void CreateJustSubcommands(CLI::App& app) {
@@ -194,6 +203,8 @@ void CreateJustSubcommands(CLI::App& app) {
         "execute", "Start single node execution service on this machine.");
     auto* cmd_serve =
         app.add_subcommand("serve", "Provide target dependencies for a build.");
+    auto* cmd_preprocess =
+        app.add_subcommand("preprocess", "Preprocess Jsonnet file.");
     auto* cmd_traverse =
         app.group("")  // group for creating hidden options
             ->add_subcommand("traverse",
@@ -212,6 +223,7 @@ void CreateJustSubcommands(CLI::App& app) {
     std::ignore = cmd_gc;
     std::ignore = cmd_execution;
     std::ignore = cmd_serve;
+    std::ignore = cmd_preprocess;
 }
 
 auto ParseCommandLineArguments(int argc, char const* const* argv)
@@ -232,6 +244,7 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
     auto cmd_gc = app.get_subcommand("gc");
     auto cmd_execution = app.get_subcommand("execute");
     auto cmd_serve = app.get_subcommand("serve");
+    auto cmd_preprocess = app.get_subcommand("preprocess");
 
     CommandLineArguments clargs;
     SetupDescribeCommandArguments(cmd_describe, &clargs);
@@ -245,6 +258,7 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
     SetupGcCommandArguments(cmd_gc, &clargs);
     SetupExecutionServiceCommandArguments(cmd_execution, &clargs);
     SetupServeServiceCommandArguments(cmd_serve, &clargs);
+    SetupPreProcessCommandArguments(cmd_preprocess, &clargs);
     try {
         app.parse(argc, argv);
     } catch (CLI::Error& e) {
@@ -290,6 +304,9 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
     else if (*cmd_serve) {
         clargs.cmd = SubCommand::kServe;
     }
+    else if (*cmd_preprocess) {
+        clargs.cmd = SubCommand::kPreProcess;
+    }
 
     return clargs;
 }
diff --git a/src/buildtool/main/cli.hpp b/src/buildtool/main/cli.hpp
--- a/src/buildtool/main/cli.hpp
+++ b/src/buildtool/main/cli.hpp
@@ -36,7 +36,8 @@ enum class SubCommand : std::uint8_t {
     kTraverse,
     kGc,
     kExecute,
-    kServe
+    kServe,
+    kPreProcess
 };
 
 struct CommandLineArguments {
@@ -57,6 +58,7 @@ struct CommandLineArguments {
     ServerAuthArguments sauth;
     ServiceArguments service;
     ServeArguments serve;
+    PreprocessArguments preprocess;
     RetryArguments retry;
     GcArguments gc;
     ToAddArguments to_add;
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -454,11 +454,12 @@ void StoreTargetCacheShard(
         .config = std::move(config)};
 }
 
-[[nodiscard]] auto DetermineWorkspaceRootByLookingForMarkers() noexcept
+[[nodiscard]] auto DetermineWorkspaceRootByLookingForMarkers(
+    std::optional<std::filesystem::path> const& base = std::nullopt) noexcept
     -> std::filesystem::path {
     std::filesystem::path cwd{};
     try {
-        cwd = std::filesystem::current_path();
+        cwd = base.value_or(std::filesystem::current_path());
     } catch (std::exception const& e) {
         Logger::Log(LogLevel::Warning,
                     "Failed to determine current working directory ({})",
@@ -920,6 +921,57 @@ auto main(int argc, char* argv[]) -> int {
             return kExitFailure;
         }
 
+        if (arguments.cmd == SubCommand::kPreProcess) {
+            auto path = arguments.preprocess.path;
+            auto root = arguments.preprocess.root;
+            std::optional<std::string> content{};
+
+            if (path == "-") {
+                path = "<stdin>";
+                if (not root) {
+                    root = std::filesystem::current_path();
+                }
+                content = "";
+                while (std::cin) {
+                    std::string line;
+                    std::getline(std::cin, line);
+                    content = fmt::format("{}{}\n", *content, line);
+                }
+            }
+            else if (not root) {
+                path = std::filesystem::absolute(path);
+                root = DetermineWorkspaceRootByLookingForMarkers(
+                    path.parent_path());
+                Logger::Log(
+                    LogLevel::Info, "Using preprocess root {}", root->string());
+                path = std::filesystem::proximate(path, *root);
+            }
+
+            FileRoot file_root{*root};
+            if (not content) {
+                content = file_root.ReadContent(path);
+                if (not content) {
+                    Logger::Log(
+                        LogLevel::Error, "Failed to read {}.", path.string());
+                    return kExitFailure;
+                }
+            }
+
+            if (auto output = file_root.ReadJSON(*content, path)) {
+                std::cout << IndentListsOnlyUntilDepth(
+                                 nlohmann::json::parse(*output), 2, 1)
+                          << std::endl;
+                return kExitSuccess;
+            }
+
+            Logger::Log(LogLevel::Error,
+                        "Failed to preprocess {} in root {}.",
+                        path.string(),
+                        root->string());
+
+            return kExitFailure;
+        }
+
         // If no execution endpoint was given, the client should default to the
         // serve endpoint, if given.
         if (not arguments.endpoint.remote_execution_address.has_value() and
diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp
--- a/src/other_tools/just_mr/utils.hpp
+++ b/src/other_tools/just_mr/utils.hpp
@@ -148,7 +148,16 @@ std::map<std::string, JustSubCmdFlags> const kKnownJustSubcommands{
       .remote = true,
       .remote_props = true,
       .serve = false,
-      .dispatch = true}}};
+      .dispatch = true}},
+    {"preprocess",
+     {.config = false,
+      .build_root = false,
+      .launch = false,
+      .defines = false,
+      .remote = false,
+      .remote_props = false,
+      .serve = false,
+      .dispatch = false}}};
 
 nlohmann::json const kDefaultConfigLocations = nlohmann::json::array(
     {{{"root", "workspace"}, {"path", "repos.json"}},
--