Merge pull request #203 from marklogic/feature/15636-disable-spark-ui

Added --spark-show-progress-bar to make it conditional

Author: rjrudin

docs/common-options.md

@@ -225,9 +225,32 @@ Flux will then log the count:
 [main] INFO com.marklogic.flux: Count: 1000
 ```
 
+## Viewing progress
+
+For commands that write data to MarkLogic - which includes all import, copy, and reprocess commands - Flux will log
+messages at the `INFO` level showing progress in terms of how much data has been sent to MarkLogic. For import and 
+copy commands, Flux defaults to logging a message every time approximately 10,000 documents are written. For the 
+reprocess command, Flux defaults to logging a message every time approximately 10,000 items are reprocessed by 
+MarkLogic. These values can be adjusted via the `--log-progress` option. The guides on import, copy, and reprocess
+provide further details on logging progress. 
+
+For export commands, Flux does not log progress as it does not always have visibility into how much data has been 
+written to an external data source (whereas it does have visibility into how much data has been written to 
+MarkLogic). You can instead include the `--spark-show-progress-bar` option to enable the progress bar provided
+by the underlying [Apache Spark software](https://spark.apache.org/). The Spark progress bar provides details at a
+lower level that may not provide visibility into the amount of data that has been exported, but it will at least avoid
+the impression that Flux is stuck. Typically, the best approach for monitoring progress for export commands 
+will be to examine the target destination to see how much data has been written. 
+
 ## Viewing a stacktrace
 
 When a command fails, Flux will stop execution of the command and display an error message. If you wish to see the 
 underlying stacktrace associated with the error, run the command with the `--stacktrace` option included. This is 
 included primarily for debugging purposes, as the stacktrace may be fairly long with only a small portion of it 
 potentially being helpful. The initial error message displayed by Flux is intended to be as helpful as possible. 
+
+## Configuring logging
+
+Flux uses a [Log4J2 properties file](https://logging.apache.org/log4j/2.x/manual/configuration.html#Properties) to 
+configure its logging. The file is located in a Flux installation at `./conf/log4j2.properties`. You are free to 
+customize this file to meet your needs for logging.

flux-cli/src/dist/conf/log4j2.properties

@@ -5,8 +5,8 @@ rootLogger = INFO, STDOUT
 appender.console.name = STDOUT
 appender.console.type = Console
 appender.console.layout.type = PatternLayout
-#appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%tn] %p %c: %m%n
-appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %c: %m%n
+#appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} [%tn] %p %c: %m%n
+appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %c: %m%n
 
 logger.marklogicclient.name=com.marklogic.client
 logger.marklogicclient.level=WARN

flux-cli/src/main/java/com/marklogic/flux/cli/Main.java

@@ -3,10 +3,7 @@
  */
 package com.marklogic.flux.cli;
 
-import com.marklogic.flux.impl.AbstractCommand;
-import com.marklogic.flux.impl.Command;
-import com.marklogic.flux.impl.SparkUtil;
-import com.marklogic.flux.impl.VersionCommand;
+import com.marklogic.flux.impl.*;
 import com.marklogic.flux.impl.copy.CopyCommand;
 import com.marklogic.flux.impl.custom.CustomExportDocumentsCommand;
 import com.marklogic.flux.impl.custom.CustomExportRowsCommand;
@@ -112,11 +109,15 @@ private int executeCommand(CommandLine.ParseResult parseResult) {
 
     protected SparkSession buildSparkSession(Command selectedCommand) {
         String masterUrl = null;
+        boolean showProgressBar = false;
         if (selectedCommand instanceof AbstractCommand) {
-            masterUrl = ((AbstractCommand) selectedCommand).getCommonParams().getSparkMasterUrl();
+            CommonParams commonParams = ((AbstractCommand) selectedCommand).getCommonParams();
+            masterUrl = commonParams.getSparkMasterUrl();
+            showProgressBar = commonParams.isSparkShowProgressBar();
         }
+
         return masterUrl != null && masterUrl.trim().length() > 0 ?
-            SparkUtil.buildSparkSession(masterUrl) :
+            SparkUtil.buildSparkSession(masterUrl, showProgressBar) :
             SparkUtil.buildSparkSession();
     }
 }

flux-cli/src/main/java/com/marklogic/flux/impl/CommonParams.java

@@ -35,14 +35,18 @@ public class CommonParams {
     @CommandLine.Option(names = "--stacktrace", description = "Print the stacktrace when a command fails.")
     private boolean showStacktrace;
 
-    // Hidden for now since showing it for every command in its "help" seems confusing for most users that will likely
-    // never need to know about this.
     @CommandLine.Option(
         names = "--spark-master-url",
         description = "Specify the Spark master URL for configuring the local Spark cluster created by Flux."
     )
     private String sparkMasterUrl = "local[*]";
 
+    @CommandLine.Option(
+        names = "--spark-show-progress-bar",
+        description = "Show the Spark progress bar in the console, which will periodically log Spark stage progress."
+    )
+    private boolean sparkShowProgressBar;
+
     @CommandLine.Option(
         names = "-C",
         description = "Specify any key and value to be added to the Spark runtime configuration; %ne.g. -Cspark.logConf=true."
@@ -82,4 +86,8 @@ public Map<String, String> getConfigParams() {
     public Preview getPreview() {
         return preview;
     }
+
+    public boolean isSparkShowProgressBar() {
+        return sparkShowProgressBar;
+    }
 }

flux-cli/src/main/java/com/marklogic/flux/impl/SparkUtil.java

@@ -16,15 +16,24 @@ private SparkUtil() {
     }
 
     public static SparkSession buildSparkSession() {
-        return buildSparkSession("local[*]");
+        return buildSparkSession("local[*]", false);
     }
 
-    public static SparkSession buildSparkSession(String masterUrl) {
-        return SparkSession.builder()
+    public static SparkSession buildSparkSession(String masterUrl, boolean showConsoleProgress) {
+        SparkSession.Builder builder = SparkSession.builder()
             .master(masterUrl)
-            .config("spark.ui.showConsoleProgress", "true")
-            .config("spark.sql.session.timeZone", "UTC")
-            .getOrCreate();
+            .config("spark.sql.session.timeZone", "UTC");
+
+        if (showConsoleProgress) {
+            // The main value of this is in showing pixels moving. The info provided by Spark - about tasks and stages -
+            // is usually going to be too low-level for a typical user. But because the Spark progress consoles shows
+            // an ASCII spinner and some things being updated, it at least gives the user comfort that Flux is not
+            // frozen. Note as well that for import, copy, and reprocess commands, the --log-progress feature provides
+            // much more useful progress status.
+            builder = builder.config("spark.ui.showConsoleProgress", "true");
+        }
+
+        return builder.getOrCreate();
     }
 
     /**

flux-cli/src/test/java/com/marklogic/flux/impl/ShowSparkProgressBarTest.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright © 2024 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
+ */
+package com.marklogic.flux.impl;
+
+import com.marklogic.flux.AbstractTest;
+import org.junit.jupiter.api.Test;
+
+class ShowSparkProgressBarTest extends AbstractTest {
+
+    @Test
+    void test() {
+        run(
+            "import-files",
+            "--path", "src/test/resources/mixed-files",
+            "--filter", "hello*",
+            "--connection-string", makeConnectionString(),
+            "--permissions", DEFAULT_PERMISSIONS,
+            "--collections", "test-files",
+            "--spark-show-progress-bar"
+        );
+
+        assertCollectionSize(
+            "This test ensures that include --spark-show-progress-bar doesn't cause any errors.",
+            "test-files", 4
+        );
+    }
+}