Skip to content

Commit 5eb2a05

Browse files
feeblefakiebrfrn169ypeckstadt
authored
Backport to branch(3.11) : [3.10, 3.11, 3.12] Backport data loader (#2364)
Co-authored-by: Toshihiro suzuki <[email protected]> Co-authored-by: Peckstadt Yves <[email protected]>
1 parent c353581 commit 5eb2a05

File tree

18 files changed

+676
-0
lines changed

18 files changed

+676
-0
lines changed

‎data-loader/build.gradle

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
subprojects {
2+
group = "scalardb.dataloader"
3+
4+
ext {
5+
apacheCommonsLangVersion = '3.14.0'
6+
apacheCommonsIoVersion = '2.16.1'
7+
}
8+
dependencies {
9+
// AssertJ
10+
testImplementation("org.assertj:assertj-core:${assertjVersion}")
11+
12+
// JUnit 5
13+
testImplementation(platform("org.junit:junit-bom:$junitVersion"))
14+
testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion")
15+
testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")
16+
testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")
17+
18+
// Apache Commons
19+
implementation("org.apache.commons:commons-lang3:${apacheCommonsLangVersion}")
20+
implementation("commons-io:commons-io:${apacheCommonsIoVersion}")
21+
22+
// Mockito
23+
testImplementation "org.mockito:mockito-core:${mockitoVersion}"
24+
testImplementation "org.mockito:mockito-inline:${mockitoVersion}"
25+
testImplementation "org.mockito:mockito-junit-jupiter:${mockitoVersion}"
26+
}
27+
}

‎data-loader/cli/build.gradle

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
plugins {
2+
id 'net.ltgt.errorprone' version "${errorpronePluginVersion}"
3+
id 'com.github.johnrengelman.shadow' version "${shadowPluginVersion}"
4+
id 'com.github.spotbugs' version "${spotbugsPluginVersion}"
5+
id 'application'
6+
}
7+
8+
application {
9+
mainClass = 'com.scalar.db.dataloader.cli.DataLoaderCli'
10+
}
11+
12+
archivesBaseName = "scalardb-data-loader-cli"
13+
14+
dependencies {
15+
implementation project(':core')
16+
implementation project(':data-loader:core')
17+
implementation "org.slf4j:slf4j-simple:${slf4jVersion}"
18+
implementation "info.picocli:picocli:${picocliVersion}"
19+
20+
// for SpotBugs
21+
compileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}"
22+
testCompileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}"
23+
24+
// for Error Prone
25+
errorprone "com.google.errorprone:error_prone_core:${errorproneVersion}"
26+
errorproneJavac "com.google.errorprone:javac:${errorproneJavacVersion}"
27+
}
28+
29+
javadoc {
30+
title = "ScalarDB Data Loader CLI"
31+
}
32+
33+
// Build a fat jar
34+
shadowJar {
35+
archiveClassifier.set("")
36+
manifest {
37+
attributes 'Main-Class': 'com.scalar.db.dataloader.DataLoaderCli'
38+
}
39+
mergeServiceFiles()
40+
}
41+
42+
spotless {
43+
java {
44+
target 'src/*/java/**/*.java'
45+
importOrder()
46+
removeUnusedImports()
47+
googleJavaFormat(googleJavaFormatVersion)
48+
}
49+
}
50+
51+
spotbugsMain.reports {
52+
html.enabled = true
53+
}
54+
spotbugsMain.excludeFilter = file("${project.rootDir}/gradle/spotbugs-exclude.xml")
55+
56+
spotbugsTest.reports {
57+
html.enabled = true
58+
}
59+
spotbugsTest.excludeFilter = file("${project.rootDir}/gradle/spotbugs-exclude.xml")

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/DataLoaderCli.java

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package com.scalar.db.dataloader.cli;
2+
3+
import com.scalar.db.dataloader.cli.command.dataexport.ExportCommand;
4+
import com.scalar.db.dataloader.cli.command.dataimport.ImportCommand;
5+
import picocli.CommandLine;
6+
7+
/** The main class to start the ScalarDB Data loader CLI tool */
8+
@CommandLine.Command(
9+
description = "ScalarDB Data Loader CLI",
10+
mixinStandardHelpOptions = true,
11+
version = "1.0",
12+
subcommands = {ImportCommand.class, ExportCommand.class})
13+
public class DataLoaderCli {
14+
15+
/**
16+
* Main method to start the ScalarDB Data Loader CLI tool
17+
*
18+
* @param args the command line arguments
19+
*/
20+
public static void main(String[] args) {
21+
int exitCode =
22+
new CommandLine(new DataLoaderCli())
23+
.setCaseInsensitiveEnumValuesAllowed(true)
24+
.execute(args);
25+
System.exit(exitCode);
26+
}
27+
}

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverter.java

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package com.scalar.db.dataloader.cli.command;
2+
3+
import com.scalar.db.dataloader.core.ColumnKeyValue;
4+
import picocli.CommandLine;
5+
6+
/**
7+
* Converts a string representation of a key-value pair into a {@link ColumnKeyValue} object. The
8+
* string format should be "key=value".
9+
*/
10+
public class ColumnKeyValueConverter implements CommandLine.ITypeConverter<ColumnKeyValue> {
11+
12+
/**
13+
* Converts a string representation of a key-value pair into a {@link ColumnKeyValue} object.
14+
*
15+
* @param keyValue the string representation of the key-value pair in the format "key=value"
16+
* @return a {@link ColumnKeyValue} object representing the key-value pair
17+
* @throws IllegalArgumentException if the input string is not in the expected format
18+
*/
19+
@Override
20+
public ColumnKeyValue convert(String keyValue) {
21+
if (keyValue == null) {
22+
throw new IllegalArgumentException("Key-value cannot be null");
23+
}
24+
String[] parts = keyValue.split("=", 2);
25+
if (parts.length != 2 || parts[0].trim().isEmpty() || parts[1].trim().isEmpty()) {
26+
throw new IllegalArgumentException("Invalid key-value format: " + keyValue);
27+
}
28+
String columnName = parts[0].trim();
29+
String value = parts[1].trim();
30+
return new ColumnKeyValue(columnName, value);
31+
}
32+
}

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package com.scalar.db.dataloader.cli.command.dataexport;
2+
3+
import com.scalar.db.dataloader.cli.exception.DirectoryValidationException;
4+
import com.scalar.db.dataloader.cli.exception.InvalidFileExtensionException;
5+
import com.scalar.db.dataloader.cli.util.DirectoryUtils;
6+
import java.io.File;
7+
import java.util.Arrays;
8+
import java.util.List;
9+
import java.util.concurrent.Callable;
10+
import javax.annotation.Nullable;
11+
import org.apache.commons.io.FilenameUtils;
12+
import org.apache.commons.lang3.StringUtils;
13+
import picocli.CommandLine;
14+
import picocli.CommandLine.Model.CommandSpec;
15+
import picocli.CommandLine.Spec;
16+
17+
@CommandLine.Command(name = "export", description = "Export data from a ScalarDB table")
18+
public class ExportCommand extends ExportCommandOptions implements Callable<Integer> {
19+
20+
private static final List<String> ALLOWED_EXTENSIONS = Arrays.asList("csv", "json", "jsonl");
21+
22+
@Spec CommandSpec spec;
23+
24+
@Override
25+
public Integer call() throws Exception {
26+
validateOutputDirectory(outputFilePath);
27+
return 0;
28+
}
29+
30+
private void validateOutputDirectory(@Nullable String path)
31+
throws DirectoryValidationException, InvalidFileExtensionException {
32+
if (path == null || path.isEmpty()) {
33+
// It is ok for the output file path to be null or empty as a default file name will be used
34+
// if not provided
35+
return;
36+
}
37+
38+
File file = new File(path);
39+
40+
if (file.isDirectory()) {
41+
validateDirectory(path);
42+
} else {
43+
validateFileExtension(file.getName());
44+
validateDirectory(file.getParent());
45+
}
46+
}
47+
48+
private void validateDirectory(String directoryPath) throws DirectoryValidationException {
49+
// If the directory path is null or empty, use the current working directory
50+
if (directoryPath == null || directoryPath.isEmpty()) {
51+
DirectoryUtils.validateTargetDirectory(DirectoryUtils.getCurrentWorkingDirectory());
52+
} else {
53+
DirectoryUtils.validateTargetDirectory(directoryPath);
54+
}
55+
}
56+
57+
private void validateFileExtension(String filename) throws InvalidFileExtensionException {
58+
String extension = FilenameUtils.getExtension(filename);
59+
if (StringUtils.isBlank(extension)) {
60+
throw new InvalidFileExtensionException(
61+
String.format("No file extension was found on the provided file name %s.", filename));
62+
}
63+
if (!ALLOWED_EXTENSIONS.contains(extension.toLowerCase())) {
64+
throw new InvalidFileExtensionException(
65+
String.format(
66+
"Invalid file extension: %s. Allowed extensions are: %s",
67+
extension, String.join(", ", ALLOWED_EXTENSIONS)));
68+
}
69+
}
70+
}

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package com.scalar.db.dataloader.cli.command.dataexport;
2+
3+
import picocli.CommandLine;
4+
5+
/** A class to represent the command options for the export command. */
6+
public class ExportCommandOptions {
7+
8+
@CommandLine.Option(
9+
names = {"--output-file", "-o"},
10+
paramLabel = "<OUTPUT_FILE>",
11+
description =
12+
"Path and name of the output file for the exported data (default: <table_name>.<format>)")
13+
protected String outputFilePath;
14+
}

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package com.scalar.db.dataloader.cli.command.dataimport;
2+
3+
import java.util.concurrent.Callable;
4+
import picocli.CommandLine;
5+
6+
@CommandLine.Command(name = "import", description = "Import data into a ScalarDB table")
7+
public class ImportCommand extends ImportCommandOptions implements Callable<Integer> {
8+
@CommandLine.Spec CommandLine.Model.CommandSpec spec;
9+
10+
@Override
11+
public Integer call() throws Exception {
12+
return 0;
13+
}
14+
}

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommandOptions.java

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
package com.scalar.db.dataloader.cli.command.dataimport;
2+
3+
public class ImportCommandOptions {}

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/DirectoryValidationException.java

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package com.scalar.db.dataloader.cli.exception;
2+
3+
/** Exception thrown when there is an error validating a directory. */
4+
public class DirectoryValidationException extends Exception {
5+
6+
public DirectoryValidationException(String message) {
7+
super(message);
8+
}
9+
}

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/InvalidFileExtensionException.java

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package com.scalar.db.dataloader.cli.exception;
2+
3+
/** An exception thrown when the file extension is invalid. */
4+
public class InvalidFileExtensionException extends Exception {
5+
6+
public InvalidFileExtensionException(String message) {
7+
super(message);
8+
}
9+
}

‎data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/DirectoryUtils.java

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package com.scalar.db.dataloader.cli.util;
2+
3+
import com.scalar.db.dataloader.cli.exception.DirectoryValidationException;
4+
import java.io.IOException;
5+
import java.nio.file.Files;
6+
import java.nio.file.Path;
7+
import java.nio.file.Paths;
8+
import org.apache.commons.lang3.StringUtils;
9+
10+
/** Utility class for validating and handling directories. */
11+
public final class DirectoryUtils {
12+
13+
private DirectoryUtils() {
14+
// restrict instantiation
15+
}
16+
17+
/**
18+
* Validates the provided directory path. Ensures that the directory exists and is writable. If
19+
* the directory doesn't exist, a creation attempt is made.
20+
*
21+
* @param directoryPath the directory path to validate
22+
* @throws DirectoryValidationException if the directory is not writable or cannot be created
23+
*/
24+
public static void validateTargetDirectory(String directoryPath)
25+
throws DirectoryValidationException {
26+
if (StringUtils.isBlank(directoryPath)) {
27+
throw new IllegalArgumentException("Directory path cannot be null or empty.");
28+
}
29+
30+
Path path = Paths.get(directoryPath);
31+
32+
if (Files.exists(path)) {
33+
// Check if the provided directory is writable
34+
if (!Files.isWritable(path)) {
35+
throw new DirectoryValidationException(
36+
String.format(
37+
"The directory '%s' does not have write permissions. Please ensure that the current user has write access to the directory.",
38+
path.toAbsolutePath()));
39+
}
40+
41+
} else {
42+
// Create the directory if it doesn't exist
43+
try {
44+
Files.createDirectories(path);
45+
} catch (IOException e) {
46+
throw new DirectoryValidationException(
47+
String.format(
48+
"Failed to create the directory '%s'. Please check if you have sufficient permissions and if there are any file system restrictions. Details: %s",
49+
path.toAbsolutePath(), e.getMessage()));
50+
}
51+
}
52+
}
53+
54+
/**
55+
* Returns the current working directory.
56+
*
57+
* @return the current working directory
58+
*/
59+
public static String getCurrentWorkingDirectory() {
60+
return Paths.get(System.getProperty("user.dir")).toAbsolutePath().toString();
61+
}
62+
}

‎data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverterTest.java

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package com.scalar.db.dataloader.cli.command;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
import static org.junit.jupiter.api.Assertions.assertThrows;
5+
6+
import com.scalar.db.dataloader.core.ColumnKeyValue;
7+
import org.junit.jupiter.api.Test;
8+
9+
class ColumnKeyValueConverterTest {
10+
11+
private final ColumnKeyValueConverter converter = new ColumnKeyValueConverter();
12+
13+
@Test
14+
void convert_ValidInput_ReturnsColumnKeyValue() {
15+
String input = "name=John Doe";
16+
ColumnKeyValue expected = new ColumnKeyValue("name", "John Doe");
17+
ColumnKeyValue result = converter.convert(input);
18+
assertEquals(expected.getColumnName(), result.getColumnName());
19+
assertEquals(expected.getColumnValue(), result.getColumnValue());
20+
}
21+
22+
@Test
23+
void convert_ValidInputWithExtraSpaces_ReturnsColumnKeyValue() {
24+
String input = " age = 25 ";
25+
ColumnKeyValue expected = new ColumnKeyValue("age", "25");
26+
ColumnKeyValue result = converter.convert(input);
27+
assertEquals(expected.getColumnName(), result.getColumnName());
28+
assertEquals(expected.getColumnValue(), result.getColumnValue());
29+
}
30+
31+
@Test
32+
void convert_InvalidInputMissingValue_ThrowsIllegalArgumentException() {
33+
String input = "name=";
34+
assertThrows(IllegalArgumentException.class, () -> converter.convert(input));
35+
}
36+
37+
@Test
38+
void convert_InvalidInputMissingKey_ThrowsIllegalArgumentException() {
39+
String input = "=John Doe";
40+
assertThrows(IllegalArgumentException.class, () -> converter.convert(input));
41+
}
42+
43+
@Test
44+
void convert_InvalidInputMissingEquals_ThrowsIllegalArgumentException() {
45+
String input = "nameJohn Doe";
46+
assertThrows(IllegalArgumentException.class, () -> converter.convert(input));
47+
}
48+
49+
@Test
50+
void convert_ValidInputMultipleEquals_Returns() {
51+
String input = "name=John=Doe";
52+
ColumnKeyValue expected = new ColumnKeyValue("name", "John=Doe");
53+
ColumnKeyValue result = converter.convert(input);
54+
assertEquals(expected.getColumnName(), result.getColumnName());
55+
assertEquals(expected.getColumnValue(), result.getColumnValue());
56+
}
57+
58+
@Test
59+
void convert_NullValue_ThrowsIllegalArgumentException() {
60+
assertThrows(IllegalArgumentException.class, () -> converter.convert(null));
61+
}
62+
}

0 commit comments

Comments
 (0)