Backport to branch(3.13) : Add Util classes for data loader (#2413)

Co-authored-by: inv-jishnu <[email protected]>
Co-authored-by: Peckstadt Yves <[email protected]>

Author: 3 people

core/src/main/java/com/scalar/db/common/error/CoreError.java

@@ -676,6 +676,8 @@ public enum CoreError implements ScalarDbError {
       "Invalid number specified for column %s in table %s in namespace %s",
       "",
       ""),
+  DATA_LOADER_ERROR_METHOD_NULL_ARGUMENT(
+      Category.USER_ERROR, "0151", "Method null argument not allowed", "", ""),
 
   //
   // Errors for the concurrency error category

data-loader/build.gradle

@@ -1,4 +1,7 @@
 subprojects {
+    ext {
+        jacksonVersion = '2.17.0'
+    }
     group = "scalardb.dataloader"
     dependencies {
         // AssertJ
@@ -13,6 +16,7 @@ subprojects {
         // Apache Commons
         implementation("org.apache.commons:commons-lang3:${commonsLangVersion}")
         implementation("commons-io:commons-io:${commonsIoVersion}")
+        implementation("org.slf4j:slf4j-simple:${slf4jVersion}")
 
         // Mockito
         testImplementation "org.mockito:mockito-core:${mockitoVersion}"
@@ -24,5 +28,11 @@ subprojects {
         annotationProcessor "org.projectlombok:lombok:${lombokVersion}"
         testCompileOnly "org.projectlombok:lombok:${lombokVersion}"
         testAnnotationProcessor "org.projectlombok:lombok:${lombokVersion}"
+
+        // Jackson
+        implementation("com.fasterxml.jackson.core:jackson-core:${jacksonVersion}")
+        implementation("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}")
+        implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+
     }
 }

data-loader/core/src/main/java/com/scalar/db/dataloader/core/exception/Base64Exception.java

@@ -0,0 +1,14 @@
+package com.scalar.db.dataloader.core.exception;
+
+/** Exception thrown when an error occurs while trying to encode or decode base64 values. */
+public class Base64Exception extends Exception {
+
+  /**
+   * Class constructor
+   *
+   * @param message Exception message
+   */
+  public Base64Exception(String message) {
+    super(message);
+  }
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/CollectionUtil.java

@@ -0,0 +1,23 @@
+package com.scalar.db.dataloader.core.util;
+
+import java.util.Collection;
+
+/** Utils for collection classes */
+public class CollectionUtil {
+
+  /**
+   * Check if lists are of same length
+   *
+   * @param collections List of collections
+   * @return collections are same length or not
+   */
+  public static boolean areSameLength(Collection<?>... collections) {
+    int n = collections[0].size();
+    for (Collection<?> c : collections) {
+      if (c.size() != n) {
+        return false;
+      }
+    }
+    return true;
+  }
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/CsvUtil.java

@@ -0,0 +1,17 @@
+package com.scalar.db.dataloader.core.util;
+
+/** Utils for csv data manipulation */
+public class CsvUtil {
+
+  /**
+   * Remove the last character in the string builder if it's a delimiter
+   *
+   * @param stringBuilder String builder instance
+   * @param delimiter Delimiter character used in the CSV content
+   */
+  public static void removeTrailingDelimiter(StringBuilder stringBuilder, String delimiter) {
+    if (stringBuilder.substring(stringBuilder.length() - 1).equals(delimiter)) {
+      stringBuilder.setLength(stringBuilder.length() - 1);
+    }
+  }
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/DebugUtil.java

@@ -0,0 +1,30 @@
+package com.scalar.db.dataloader.core.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DebugUtil {
+
+  private static final Logger logger = LoggerFactory.getLogger(DebugUtil.class);
+
+  /**
+   * log memory usage
+   *
+   * @param stage stage of process
+   */
+  public static void logMemoryUsage(String stage) {
+    Runtime runtime = Runtime.getRuntime();
+    long usedMemory = runtime.totalMemory() - runtime.freeMemory();
+    long maxMemory = runtime.maxMemory();
+
+    logger.info(
+        "Memory usage at {}: Used Memory = {} MB, Max Memory = {} MB",
+        stage,
+        formatMemorySize(usedMemory),
+        formatMemorySize(maxMemory));
+  }
+
+  private static String formatMemorySize(long size) {
+    return String.format("%.2f", size / (1024.0 * 1024.0));
+  }
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/DecimalUtil.java

@@ -0,0 +1,40 @@
+package com.scalar.db.dataloader.core.util;
+
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.Locale;
+
+/** Utils for decimal handling */
+public class DecimalUtil {
+
+  /**
+   * Convert a Double to a non-scientific formatted string
+   *
+   * @param doubleValue Double value
+   * @return formatted double as a string
+   */
+  public static String convertToNonScientific(Double doubleValue) {
+    return createFormatter().format(doubleValue);
+  }
+
+  /**
+   * Convert a Float to a non-scientific formatted string
+   *
+   * @param floatValue Float value
+   * @return formatted float as a string
+   */
+  public static String convertToNonScientific(Float floatValue) {
+    return createFormatter().format(floatValue);
+  }
+
+  /**
+   * Create a Decimal formatter
+   *
+   * @return decimal formatter instance
+   */
+  private static DecimalFormat createFormatter() {
+    DecimalFormat df = new DecimalFormat("0", DecimalFormatSymbols.getInstance(Locale.ENGLISH));
+    df.setMaximumFractionDigits(340); // 340 = DecimalFormat.DOUBLE_FRACTION_DIGITS
+    return df;
+  }
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/PathUtil.java

@@ -0,0 +1,24 @@
+package com.scalar.db.dataloader.core.util;
+
+import java.io.File;
+
+public class PathUtil {
+
+  /**
+   * Ensures the specified path has a trailing path separator.
+   *
+   * @param path the path
+   * @return the path with a trailing path separator.
+   */
+  public static String ensureTrailingSeparator(String path) {
+    if (path == null || path.isEmpty()) {
+      return "";
+    }
+
+    if (!path.endsWith(File.separator)) {
+      return path + File.separator;
+    }
+
+    return path;
+  }
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/RuntimeUtil.java

@@ -0,0 +1,21 @@
+package com.scalar.db.dataloader.core.util;
+
+import static com.scalar.db.common.error.CoreError.DATA_LOADER_ERROR_METHOD_NULL_ARGUMENT;
+
+/** Utils for runtime checks */
+public class RuntimeUtil {
+
+  /**
+   * Argument null check
+   *
+   * @param values List of arguments
+   * @throws NullPointerException when one of the arguments is null
+   */
+  public static void checkNotNull(Object... values) {
+    for (Object value : values) {
+      if (value == null) {
+        throw new NullPointerException(DATA_LOADER_ERROR_METHOD_NULL_ARGUMENT.buildMessage());
+      }
+    }
+  }
+}