diff --git a/core/src/main/java/com/scalar/db/common/error/CoreError.java b/core/src/main/java/com/scalar/db/common/error/CoreError.java index eba2ea3256..9483a446dd 100644 --- a/core/src/main/java/com/scalar/db/common/error/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/error/CoreError.java @@ -826,6 +826,21 @@ public enum CoreError implements ScalarDbError { DATA_LOADER_FILE_PATH_IS_BLANK( Category.USER_ERROR, "0197", "File path must not be blank.", "", ""), DATA_LOADER_FILE_NOT_FOUND(Category.USER_ERROR, "0198", "File not found: %s", "", ""), + DATA_LOADER_INVALID_DATE_TIME_FOR_COLUMN_VALUE( + Category.USER_ERROR, + "0199", + "Invalid date time value specified for column %s in table %s in namespace %s.", + "", + ""), + DATA_LOADER_NULL_OR_EMPTY_KEY_VALUE_INPUT( + Category.USER_ERROR, "0200", "Key-value cannot be null or empty", "", ""), + DATA_LOADER_INVALID_KEY_VALUE_INPUT( + Category.USER_ERROR, "0201", "Invalid key-value format: %s", "", ""), + DATA_LOADER_SPLIT_INPUT_VALUE_NULL(Category.USER_ERROR, "0202", "Value must not be null", "", ""), + DATA_LOADER_SPLIT_INPUT_DELIMITER_NULL( + Category.USER_ERROR, "0203", "Delimiter must not be null", "", ""), + DATA_LOADER_CONFIG_FILE_PATH_BLANK( + Category.USER_ERROR, "0204", "Config file path must not be blank", "", ""), // // Errors for the concurrency error category diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java old mode 100644 new mode 100755 index 4c88be4594..fdedbeef2c --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -1,70 +1,207 @@ package com.scalar.db.dataloader.cli.command.dataexport; +import static java.nio.file.StandardOpenOption.APPEND; +import static java.nio.file.StandardOpenOption.CREATE; + +import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.TableMetadata; import com.scalar.db.common.error.CoreError; import com.scalar.db.dataloader.cli.exception.DirectoryValidationException; -import com.scalar.db.dataloader.cli.exception.InvalidFileExtensionException; import com.scalar.db.dataloader.cli.util.DirectoryUtils; -import java.io.File; -import java.util.Arrays; +import com.scalar.db.dataloader.cli.util.FileUtils; +import com.scalar.db.dataloader.cli.util.InvalidFilePathException; +import com.scalar.db.dataloader.core.ColumnKeyValue; +import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScanRange; +import com.scalar.db.dataloader.core.dataexport.CsvExportManager; +import com.scalar.db.dataloader.core.dataexport.ExportManager; +import com.scalar.db.dataloader.core.dataexport.ExportOptions; +import com.scalar.db.dataloader.core.dataexport.JsonExportManager; +import com.scalar.db.dataloader.core.dataexport.JsonLineExportManager; +import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; +import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; +import com.scalar.db.dataloader.core.exception.ColumnParsingException; +import com.scalar.db.dataloader.core.exception.KeyParsingException; +import com.scalar.db.dataloader.core.tablemetadata.TableMetadataException; +import com.scalar.db.dataloader.core.tablemetadata.TableMetadataService; +import com.scalar.db.dataloader.core.util.KeyUtils; +import com.scalar.db.io.Key; +import com.scalar.db.service.StorageFactory; +import java.io.BufferedWriter; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.List; +import java.util.Objects; import java.util.concurrent.Callable; -import javax.annotation.Nullable; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import picocli.CommandLine; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Spec; -@CommandLine.Command(name = "export", description = "Export data from a ScalarDB table") +@CommandLine.Command(name = "export", description = "export data from a ScalarDB table") public class ExportCommand extends ExportCommandOptions implements Callable { - private static final List ALLOWED_EXTENSIONS = Arrays.asList("csv", "json", "jsonl"); + private static final String EXPORT_FILE_NAME_FORMAT = "export.%s.%s.%s.%s"; + private static final Logger logger = LoggerFactory.getLogger(ExportCommand.class); @Spec CommandSpec spec; @Override public Integer call() throws Exception { - validateOutputDirectory(outputFilePath); + String scalarDbPropertiesFilePath = getScalarDbPropertiesFilePath(); + + try { + validateOutputDirectory(); + FileUtils.validateFilePath(scalarDbPropertiesFilePath); + + StorageFactory storageFactory = StorageFactory.create(scalarDbPropertiesFilePath); + TableMetadataService metaDataService = + new TableMetadataService(storageFactory.getStorageAdmin()); + ScalarDbDao scalarDbDao = new ScalarDbDao(); + + ExportManager exportManager = createExportManager(storageFactory, scalarDbDao, outputFormat); + + TableMetadata tableMetadata = metaDataService.getTableMetadata(namespace, table); + + Key partitionKey = + partitionKeyValue != null ? getKeysFromList(partitionKeyValue, tableMetadata) : null; + Key scanStartKey = + scanStartKeyValue != null + ? getKey(scanStartKeyValue, namespace, table, tableMetadata) + : null; + Key scanEndKey = + scanEndKeyValue != null ? getKey(scanEndKeyValue, namespace, table, tableMetadata) : null; + + ScanRange scanRange = + new ScanRange(scanStartKey, scanEndKey, scanStartInclusive, scanEndInclusive); + ExportOptions exportOptions = buildExportOptions(partitionKey, scanRange); + + String filePath = + getOutputAbsoluteFilePath( + outputDirectory, outputFileName, exportOptions.getOutputFileFormat()); + logger.info("Exporting data to file: {}", filePath); + + try (BufferedWriter writer = + Files.newBufferedWriter(Paths.get(filePath), Charset.defaultCharset(), CREATE, APPEND)) { + exportManager.startExport(exportOptions, tableMetadata, writer); + } + + } catch (DirectoryValidationException e) { + logger.error("Invalid output directory path: {}", outputDirectory); + return 1; + } catch (InvalidFilePathException e) { + logger.error( + "The ScalarDB connection settings file path is invalid or the file is missing: {}", + scalarDbPropertiesFilePath); + return 1; + } catch (TableMetadataException e) { + logger.error("Failed to retrieve table metadata: {}", e.getMessage()); + return 1; + } return 0; } - private void validateOutputDirectory(@Nullable String path) - throws DirectoryValidationException, InvalidFileExtensionException { - if (path == null || path.isEmpty()) { - // It is ok for the output file path to be null or empty as a default file name will be used - // if not provided - return; + private String getScalarDbPropertiesFilePath() { + if (StringUtils.isBlank(configFilePath)) { + throw new IllegalArgumentException( + CoreError.DATA_LOADER_CONFIG_FILE_PATH_BLANK.buildMessage()); } + return Objects.equals(configFilePath, DEFAULT_CONFIG_FILE_NAME) + ? Paths.get("").toAbsolutePath().resolve(DEFAULT_CONFIG_FILE_NAME).toString() + : configFilePath; + } - File file = new File(path); - - if (file.isDirectory()) { - validateDirectory(path); + private void validateOutputDirectory() throws DirectoryValidationException { + if (StringUtils.isBlank(outputDirectory)) { + DirectoryUtils.validateWorkingDirectory(); } else { - validateFileExtension(file.getName()); - validateDirectory(file.getParent()); + DirectoryUtils.validateOrCreateTargetDirectory(outputDirectory); } } - private void validateDirectory(String directoryPath) throws DirectoryValidationException { - // If the directory path is null or empty, use the current working directory - if (directoryPath == null || directoryPath.isEmpty()) { - DirectoryUtils.validateOrCreateTargetDirectory(DirectoryUtils.getCurrentWorkingDirectory()); - } else { - DirectoryUtils.validateOrCreateTargetDirectory(directoryPath); + private ExportManager createExportManager( + StorageFactory storageFactory, ScalarDbDao scalarDbDao, FileFormat fileFormat) { + ProducerTaskFactory taskFactory = + new ProducerTaskFactory(delimiter, includeTransactionMetadata, prettyPrintJson); + DistributedStorage storage = storageFactory.getStorage(); + switch (fileFormat) { + case JSON: + return new JsonExportManager(storage, scalarDbDao, taskFactory); + case JSONL: + return new JsonLineExportManager(storage, scalarDbDao, taskFactory); + case CSV: + return new CsvExportManager(storage, scalarDbDao, taskFactory); + default: + throw new AssertionError("Invalid file format" + fileFormat); } } - private void validateFileExtension(String filename) throws InvalidFileExtensionException { - String extension = FilenameUtils.getExtension(filename); - if (StringUtils.isBlank(extension)) { - throw new InvalidFileExtensionException( - CoreError.DATA_LOADER_MISSING_FILE_EXTENSION.buildMessage(filename)); + private ExportOptions buildExportOptions(Key partitionKey, ScanRange scanRange) { + ExportOptions.ExportOptionsBuilder builder = + ExportOptions.builder(namespace, table, partitionKey, outputFormat) + .sortOrders(sortOrders) + .excludeHeaderRow(excludeHeader) + .includeTransactionMetadata(includeTransactionMetadata) + .delimiter(delimiter) + .limit(limit) + .maxThreadCount(maxThreads) + .dataChunkSize(dataChunkSize) + .prettyPrintJson(prettyPrintJson) + .scanRange(scanRange); + + if (projectionColumns != null) { + builder.projectionColumns(projectionColumns); } - if (!ALLOWED_EXTENSIONS.contains(extension.toLowerCase())) { - throw new InvalidFileExtensionException( - CoreError.DATA_LOADER_INVALID_FILE_EXTENSION.buildMessage( - extension, String.join(", ", ALLOWED_EXTENSIONS))); + + return builder.build(); + } + + private String getOutputAbsoluteFilePath( + String outputDirectory, String outputFileName, FileFormat outputFormat) { + String fileName = + StringUtils.isBlank(outputFileName) + ? String.format( + EXPORT_FILE_NAME_FORMAT, + namespace, + table, + System.nanoTime(), + outputFormat.toString().toLowerCase()) + : outputFileName; + + if (StringUtils.isBlank(outputDirectory)) { + return Paths.get("").toAbsolutePath().resolve(fileName).toAbsolutePath().toString(); + } else { + return Paths.get(outputDirectory).resolve(fileName).toAbsolutePath().toString(); } } + + /** + * Convert ColumnKeyValue list to a key + * + * @param keyValueList key value list + * @param tableMetadata table metadata + * @return key + * @throws ColumnParsingException if any error occur during parsing column value + */ + private Key getKeysFromList(List keyValueList, TableMetadata tableMetadata) + throws ColumnParsingException { + return KeyUtils.parseMultipleKeyValues(keyValueList, tableMetadata); + } + + /** + * Convert ColumnKeyValue to a key + * + * @param keyValue key value + * @param tableMetadata table metadata + * @return key + * @throws KeyParsingException if any error occur during decoding key + */ + private Key getKey( + ColumnKeyValue keyValue, String namespace, String table, TableMetadata tableMetadata) + throws KeyParsingException { + return KeyUtils.parseKeyValue(keyValue, namespace, table, tableMetadata); + } } diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java old mode 100644 new mode 100755 index b5ac608f27..5cbe8f6c78 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java @@ -1,14 +1,147 @@ package com.scalar.db.dataloader.cli.command.dataexport; +import com.scalar.db.api.Scan; +import com.scalar.db.dataloader.core.ColumnKeyValue; +import com.scalar.db.dataloader.core.FileFormat; +import java.util.ArrayList; +import java.util.List; import picocli.CommandLine; -/** A class to represent the command options for the export command. */ public class ExportCommandOptions { + protected static final String DEFAULT_CONFIG_FILE_NAME = "scalardb.properties"; + + @CommandLine.Option( + names = {"--config", "-c"}, + paramLabel = "", + description = "Path to the ScalarDB configuration file (default: scalardb.properties)", + defaultValue = DEFAULT_CONFIG_FILE_NAME) + protected String configFilePath; + + @CommandLine.Option( + names = {"--namespace", "-ns"}, + paramLabel = "", + required = true, + description = "ScalarDB namespace containing the table to export data from") + protected String namespace; + + @CommandLine.Option( + names = {"--table", "-t"}, + paramLabel = "", + required = true, + description = "Name of the ScalarDB table to export data from") + protected String table; + @CommandLine.Option( names = {"--output-file", "-o"}, - paramLabel = "", + paramLabel = "", description = - "Path and name of the output file for the exported data (default: .)") - protected String outputFilePath; + "Name of the output file for the exported data (default: export..
..)") + protected String outputFileName; + + @CommandLine.Option( + names = {"--output-dir", "-d"}, + paramLabel = "", + description = + "Directory where the exported file should be saved (default: current directory)") + protected String outputDirectory; + + @CommandLine.Option( + names = {"--partition-key", "-pk"}, + paramLabel = "", + description = "ScalarDB partition key and value in the format 'key=value'", + converter = MultiColumnKeyValueConverter.class) + protected List partitionKeyValue; + + @CommandLine.Option( + names = {"--format", "-fmt"}, + paramLabel = "", + description = "Format of the exported data file (json, csv, jsonl) (default: json)", + defaultValue = "json") + protected FileFormat outputFormat; + + @CommandLine.Option( + names = {"--include-metadata", "-m"}, + description = "Include transaction metadata in the exported data (default: false)", + defaultValue = "false") + protected boolean includeTransactionMetadata; + + @CommandLine.Option( + names = {"--max-threads", "-mt"}, + paramLabel = "", + description = + "Maximum number of threads to use for parallel processing (default: number of available processors)") + protected int maxThreads; + + @CommandLine.Option( + names = {"--start-key", "-sk"}, + paramLabel = "", + description = "Clustering key and value to mark the start of the scan (inclusive)", + converter = SingleColumnKeyValueConverter.class) + protected ColumnKeyValue scanStartKeyValue; + + @CommandLine.Option( + names = {"--start-inclusive", "-si"}, + description = "Make the start key inclusive (default: true)", + defaultValue = "true") + // TODO: test that -si false, works + protected boolean scanStartInclusive; + + @CommandLine.Option( + names = {"--end-key", "-ek"}, + paramLabel = "", + description = "Clustering key and value to mark the end of the scan (inclusive)", + converter = SingleColumnKeyValueConverter.class) + protected ColumnKeyValue scanEndKeyValue; + + @CommandLine.Option( + names = {"--end-inclusive", "-ei"}, + description = "Make the end key inclusive (default: true)", + defaultValue = "true") + protected boolean scanEndInclusive; + + @CommandLine.Option( + names = {"--sort-by", "-s"}, + paramLabel = "", + description = "Clustering key sorting order (asc, desc)", + converter = ScanOrderingConverter.class) + protected List sortOrders = new ArrayList<>(); + + @CommandLine.Option( + names = {"--projection", "-p"}, + paramLabel = "", + description = "Columns to include in the export (comma-separated)", + split = ",") + protected List projectionColumns; + + @CommandLine.Option( + names = {"--limit", "-l"}, + paramLabel = "", + description = "Maximum number of rows to export") + protected int limit; + + @CommandLine.Option( + names = {"--delimiter"}, + paramLabel = "", + defaultValue = ",", + description = "Delimiter character for CSV files (default: comma)") + protected String delimiter; + + @CommandLine.Option( + names = {"--no-header", "-nh"}, + description = "Exclude header row in CSV files (default: false)", + defaultValue = "false") + protected boolean excludeHeader; + + @CommandLine.Option( + names = {"--pretty-print", "-pp"}, + description = "Pretty-print JSON output (default: false)", + defaultValue = "false") + protected boolean prettyPrintJson; + + @CommandLine.Option( + names = {"--data-chunk-size", "-dcs"}, + description = "Size of the data chunk to process in a single task (default: 200)", + defaultValue = "200") + protected int dataChunkSize; } diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/MultiColumnKeyValueConverter.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/MultiColumnKeyValueConverter.java new file mode 100755 index 0000000000..6479089f18 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/MultiColumnKeyValueConverter.java @@ -0,0 +1,47 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.cli.util.CommandLineInputUtils; +import com.scalar.db.dataloader.core.ColumnKeyValue; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import picocli.CommandLine; + +/** + * Converts a string representation of multiple key-value pairs into a list of {@link + * ColumnKeyValue} objects. + * + *

The expected format for the input string is: {@code key1=value1,key2=value2,...}. Each + * key-value pair should be separated by a comma, and each pair must follow the "key=value" format. + * + *

Example usage: + * + * 

+ *   MultiColumnKeyValueConverter converter = new MultiColumnKeyValueConverter();
+ *   List<ColumnKeyValue> result = converter.convert("name=John,age=30,city=New York");
+ *
+ */ +public class MultiColumnKeyValueConverter + implements CommandLine.ITypeConverter> { + + /** + * Converts a comma-separated string of key-value pairs into a list of {@link ColumnKeyValue} + * objects. + * + * @param keyValue the input string in the format {@code key1=value1,key2=value2,...} + * @return a list of {@link ColumnKeyValue} objects representing the parsed key-value pairs + * @throws IllegalArgumentException if the input is null, empty, or contains invalid formatting + */ + @Override + public List convert(String keyValue) { + if (keyValue == null || keyValue.trim().isEmpty()) { + throw new IllegalArgumentException( + CoreError.DATA_LOADER_NULL_OR_EMPTY_KEY_VALUE_INPUT.buildMessage()); + } + return Arrays.stream(CommandLineInputUtils.splitByDelimiter(keyValue, ",", 0)) + .map(CommandLineInputUtils::parseKeyValue) + .map(entry -> new ColumnKeyValue(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ScanOrderingConverter.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ScanOrderingConverter.java new file mode 100755 index 0000000000..44267017ca --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ScanOrderingConverter.java @@ -0,0 +1,33 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import com.scalar.db.api.Scan; +import com.scalar.db.dataloader.cli.util.CommandLineInputUtils; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import picocli.CommandLine; + +public class ScanOrderingConverter implements CommandLine.ITypeConverter> { + /** + * Converts a comma-separated string of key-value pairs into a list of {@link Scan.Ordering} + * objects. Each pair must be in the format "column=order", where "order" is a valid enum value of + * {@link Scan.Ordering.Order} (e.g., ASC or DESC, case-insensitive). + * + * @param value the comma-separated key-value string to convert + * @return a list of {@link Scan.Ordering} objects constructed from the input + * @throws IllegalArgumentException if parsing fails due to invalid format or enum value + */ + @Override + public List convert(String value) { + return Arrays.stream(CommandLineInputUtils.splitByDelimiter(value, ",", 0)) + .map(CommandLineInputUtils::parseKeyValue) + .map( + entry -> { + String columnName = entry.getKey(); + Scan.Ordering.Order sortOrder = + Scan.Ordering.Order.valueOf(entry.getValue().trim().toUpperCase()); + return new Scan.Ordering(columnName, sortOrder); + }) + .collect(Collectors.toList()); + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/SingleColumnKeyValueConverter.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/SingleColumnKeyValueConverter.java new file mode 100755 index 0000000000..b96e48247d --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/SingleColumnKeyValueConverter.java @@ -0,0 +1,27 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import static com.scalar.db.dataloader.cli.util.CommandLineInputUtils.parseKeyValue; + +import com.scalar.db.dataloader.core.ColumnKeyValue; +import java.util.Map; +import picocli.CommandLine; + +/** + * Converts a string representation of a key-value pair into a {@link ColumnKeyValue} object. The + * string format should be "key=value". + */ +public class SingleColumnKeyValueConverter implements CommandLine.ITypeConverter { + + /** + * Converts a string representation of a key-value pair into a {@link ColumnKeyValue} object. + * + * @param keyValue the string representation of the key-value pair in the format "key=value" + * @return a {@link ColumnKeyValue} object representing the key-value pair + * @throws IllegalArgumentException if the input string is not in the expected format + */ + @Override + public ColumnKeyValue convert(String keyValue) { + Map.Entry data = parseKeyValue(keyValue); + return new ColumnKeyValue(data.getKey(), data.getValue()); + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/CommandLineInputUtils.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/CommandLineInputUtils.java new file mode 100644 index 0000000000..88e2188656 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/CommandLineInputUtils.java @@ -0,0 +1,49 @@ +package com.scalar.db.dataloader.cli.util; + +import com.scalar.db.common.error.CoreError; +import java.util.AbstractMap; +import java.util.Map; +import java.util.Objects; +import org.apache.commons.lang3.StringUtils; + +public class CommandLineInputUtils { + + /** + * Parses a single key-value pair from a string in the format "key=value". + * + * @param keyValue the key-value string to parse + * @return a {@link Map.Entry} representing the parsed key-value pair + * @throws IllegalArgumentException if the input is null, empty, or not in the expected format + */ + public static Map.Entry parseKeyValue(String keyValue) { + if (StringUtils.isBlank(keyValue)) { + throw new IllegalArgumentException( + CoreError.DATA_LOADER_NULL_OR_EMPTY_KEY_VALUE_INPUT.buildMessage()); + } + + String[] parts = splitByDelimiter(keyValue, "=", 2); + + if (parts.length != 2 || parts[0].trim().isEmpty() || parts[1].trim().isEmpty()) { + throw new IllegalArgumentException( + CoreError.DATA_LOADER_INVALID_KEY_VALUE_INPUT.buildMessage(keyValue)); + } + return new AbstractMap.SimpleEntry<>(parts[0].trim(), parts[1].trim()); + } + + /** + * Splits a string based on the provided delimiter. + * + * @param value the string to split + * @param delimiter the delimiter to use + * @param limit the maximum number of elements in the result (same behavior as String.split() with + * limit) + * @return an array of split values + * @throws NullPointerException if value or delimiter is null + */ + public static String[] splitByDelimiter(String value, String delimiter, int limit) { + Objects.requireNonNull(value, CoreError.DATA_LOADER_SPLIT_INPUT_VALUE_NULL.buildMessage()); + Objects.requireNonNull( + delimiter, CoreError.DATA_LOADER_SPLIT_INPUT_DELIMITER_NULL.buildMessage()); + return value.split(delimiter, limit); + } +} diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java old mode 100644 new mode 100755 index 538de9f404..73934f340d --- a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java @@ -1,112 +1,56 @@ package com.scalar.db.dataloader.cli.command.dataexport; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertThrows; -import com.scalar.db.dataloader.cli.exception.InvalidFileExtensionException; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.FileFormat; +import java.io.File; import java.nio.file.Paths; -import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine; class ExportCommandTest { - private static final Logger LOGGER = LoggerFactory.getLogger(ExportCommandTest.class); - @TempDir Path tempDir; - - private ExportCommand exportCommand; - - @BeforeEach - void setUp() { - exportCommand = new ExportCommand(); - CommandLine cmd = new CommandLine(exportCommand); - exportCommand.spec = cmd.getCommandSpec(); - } - @AfterEach - public void cleanup() throws IOException { - cleanUpTempDir(); - } - - @Test - void call_WithValidOutputDirectory_ShouldReturnZero() throws Exception { - Path configFile = tempDir.resolve("config.properties"); - Files.createFile(configFile); - - Path outputDir = tempDir.resolve("output"); - Files.createDirectory(outputDir); - - exportCommand.outputFilePath = outputDir.toString(); - - assertEquals(0, exportCommand.call()); - } - - @Test - void call_WithInvalidOutputDirectory_ShouldThrowInvalidFileExtensionException() - throws IOException { - Path configFile = tempDir.resolve("config.properties"); - Files.createFile(configFile); - - Path outputDir = tempDir.resolve("output"); - outputDir.toFile().setWritable(false); - - exportCommand.outputFilePath = outputDir.toString(); - - assertThrows(InvalidFileExtensionException.class, () -> exportCommand.call()); - } - - @Test - void call_WithValidOutputFile_ShouldReturnZero() throws Exception { - Path configFile = tempDir.resolve("config.properties"); - Files.createFile(configFile); - - Path outputFile = tempDir.resolve("output.csv"); - - exportCommand.outputFilePath = outputFile.toString(); - - assertEquals(0, exportCommand.call()); + void removeFileIfCreated() { + // To remove generated file if it is present + String filePath = Paths.get("").toAbsolutePath() + "/sample.json"; + File file = new File(filePath); + if (file.exists()) { + file.deleteOnExit(); + } } @Test - void call_WithValidOutputFileInCurrentDirectory_ShouldReturnZero() throws Exception { - Path configFile = tempDir.resolve("config.properties"); - Files.createFile(configFile); - - Path outputFile = Paths.get("output.csv"); - - exportCommand.outputFilePath = outputFile.toString(); - - assertEquals(0, exportCommand.call()); + void call_withBlankScalarDBConfigurationFile_shouldThrowException() { + ExportCommand exportCommand = new ExportCommand(); + exportCommand.configFilePath = ""; + exportCommand.dataChunkSize = 100; + exportCommand.namespace = "scalar"; + exportCommand.table = "asset"; + exportCommand.outputDirectory = ""; + exportCommand.outputFileName = "sample.json"; + exportCommand.outputFormat = FileFormat.JSON; + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + exportCommand::call, + "Expected to throw FileNotFound exception as configuration path is invalid"); + Assertions.assertEquals( + CoreError.DATA_LOADER_CONFIG_FILE_PATH_BLANK.buildMessage(), thrown.getMessage()); } @Test - void call_WithValidOutputFileWithoutDirectory_ShouldReturnZero() throws Exception { - Path configFile = tempDir.resolve("config.properties"); - Files.createFile(configFile); - - exportCommand.outputFilePath = "output.csv"; - - assertEquals(0, exportCommand.call()); - } - - private void cleanUpTempDir() throws IOException { - try (Stream paths = Files.list(tempDir)) { - paths.forEach(this::deleteFile); - } - } - - private void deleteFile(Path file) { - try { - Files.deleteIfExists(file); - } catch (IOException e) { - LOGGER.error("Failed to delete file: {}", file, e); - } + void call_withInvalidScalarDBConfigurationFile_shouldReturnOne() throws Exception { + ExportCommand exportCommand = new ExportCommand(); + exportCommand.configFilePath = "scalardb.properties"; + exportCommand.dataChunkSize = 100; + exportCommand.namespace = "scalar"; + exportCommand.table = "asset"; + exportCommand.outputDirectory = ""; + exportCommand.outputFileName = "sample.json"; + exportCommand.outputFormat = FileFormat.JSON; + Assertions.assertEquals(1, exportCommand.call()); } } diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/MultiColumnKeyValueConverterTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/MultiColumnKeyValueConverterTest.java new file mode 100755 index 0000000000..89adb8f077 --- /dev/null +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/MultiColumnKeyValueConverterTest.java @@ -0,0 +1,38 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.ColumnKeyValue; +import java.util.Collections; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class MultiColumnKeyValueConverterTest { + + MultiColumnKeyValueConverter multiColumnKeyValueConverter = new MultiColumnKeyValueConverter(); + + @Test + void convert_withInvalidValue_ShouldThrowError() { + String value = "id 15"; + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> multiColumnKeyValueConverter.convert(value), + "Expected to throw exception"); + Assertions.assertEquals( + CoreError.DATA_LOADER_INVALID_KEY_VALUE_INPUT.buildMessage("id 15"), thrown.getMessage()); + } + + @Test + void convert_withValidValue_ShouldReturnColumnKeyValue() { + String value = "id=15"; + ColumnKeyValue expectedOrder = new ColumnKeyValue("id", "15"); + Assertions.assertEquals( + Collections.singletonList(expectedOrder).get(0).getColumnName(), + multiColumnKeyValueConverter.convert(value).get(0).getColumnName()); + Assertions.assertEquals( + Collections.singletonList(expectedOrder).get(0).getColumnValue(), + multiColumnKeyValueConverter.convert(value).get(0).getColumnValue()); + } +} diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ScanOrderingConverterTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ScanOrderingConverterTest.java new file mode 100755 index 0000000000..e8836ae156 --- /dev/null +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ScanOrderingConverterTest.java @@ -0,0 +1,44 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.scalar.db.api.Scan; +import com.scalar.db.common.error.CoreError; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ScanOrderingConverterTest { + ScanOrderingConverter scanOrderingConverter = new ScanOrderingConverter(); + + @Test + void callConvert_withInvalidValue_shouldThrowException() { + String value = "id ASC"; + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> scanOrderingConverter.convert(value), + "Expected to throw exception"); + Assertions.assertEquals( + CoreError.DATA_LOADER_INVALID_KEY_VALUE_INPUT.buildMessage(value), thrown.getMessage()); + } + + @Test + void callConvert_withValidValueAndOrderAscending_shouldReturnScanOrdering() { + String value = "id=ASC,age=DESC"; + List expectedOrder = new ArrayList<>(); + expectedOrder.add(new Scan.Ordering("id", Scan.Ordering.Order.ASC)); + expectedOrder.add(new Scan.Ordering("age", Scan.Ordering.Order.DESC)); + Assertions.assertEquals(expectedOrder, scanOrderingConverter.convert(value)); + } + + @Test + void callConvert_withValidValueAndOrderDescending_shouldReturnScanOrdering() { + String value = "id=desc"; + List expectedOrder = + Collections.singletonList(new Scan.Ordering("id", Scan.Ordering.Order.DESC)); + Assertions.assertEquals(expectedOrder, scanOrderingConverter.convert(value)); + } +} diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/SingleColumnKeyValueConverterTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/SingleColumnKeyValueConverterTest.java new file mode 100755 index 0000000000..0cfc894383 --- /dev/null +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/SingleColumnKeyValueConverterTest.java @@ -0,0 +1,62 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.scalar.db.dataloader.core.ColumnKeyValue; +import org.junit.jupiter.api.Test; + +class SingleColumnKeyValueConverterTest { + + private final SingleColumnKeyValueConverter converter = new SingleColumnKeyValueConverter(); + + @Test + void convert_ValidInput_ReturnsColumnKeyValue() { + String input = "name=John Doe"; + ColumnKeyValue expected = new ColumnKeyValue("name", "John Doe"); + ColumnKeyValue result = converter.convert(input); + assertEquals(expected.getColumnName(), result.getColumnName()); + assertEquals(expected.getColumnValue(), result.getColumnValue()); + } + + @Test + void convert_ValidInputWithExtraSpaces_ReturnsColumnKeyValue() { + String input = " age = 25 "; + ColumnKeyValue expected = new ColumnKeyValue("age", "25"); + ColumnKeyValue result = converter.convert(input); + assertEquals(expected.getColumnName(), result.getColumnName()); + assertEquals(expected.getColumnValue(), result.getColumnValue()); + } + + @Test + void convert_InvalidInputMissingValue_ThrowsIllegalArgumentException() { + String input = "name="; + assertThrows(IllegalArgumentException.class, () -> converter.convert(input)); + } + + @Test + void convert_InvalidInputMissingKey_ThrowsIllegalArgumentException() { + String input = "=John Doe"; + assertThrows(IllegalArgumentException.class, () -> converter.convert(input)); + } + + @Test + void convert_InvalidInputMissingEquals_ThrowsIllegalArgumentException() { + String input = "nameJohn Doe"; + assertThrows(IllegalArgumentException.class, () -> converter.convert(input)); + } + + @Test + void convert_ValidInputMultipleEquals_Returns() { + String input = "name=John=Doe"; + ColumnKeyValue expected = new ColumnKeyValue("name", "John=Doe"); + ColumnKeyValue result = converter.convert(input); + assertEquals(expected.getColumnName(), result.getColumnName()); + assertEquals(expected.getColumnValue(), result.getColumnValue()); + } + + @Test + void convert_NullValue_ThrowsIllegalArgumentException() { + assertThrows(IllegalArgumentException.class, () -> converter.convert(null)); + } +} diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/CommandLineInputUtilsTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/CommandLineInputUtilsTest.java new file mode 100644 index 0000000000..7faebef61a --- /dev/null +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/CommandLineInputUtilsTest.java @@ -0,0 +1,100 @@ +package com.scalar.db.dataloader.cli.util; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.scalar.db.common.error.CoreError; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class CommandLineInputUtilsTest { + + @Test + public void parseKeyValue_validKeyValue_ShouldReturnEntry() { + Map.Entry result = CommandLineInputUtils.parseKeyValue("foo=bar"); + + assertEquals("foo", result.getKey()); + assertEquals("bar", result.getValue()); + } + + @Test + public void parseKeyValue_whitespaceTrimmed_ShouldReturnTrimmedEntry() { + Map.Entry result = CommandLineInputUtils.parseKeyValue(" key = value "); + + assertEquals("key", result.getKey()); + assertEquals("value", result.getValue()); + } + + @Test + public void parseKeyValue_nullInput_ShouldThrowException() { + assertThrows(IllegalArgumentException.class, () -> CommandLineInputUtils.parseKeyValue(null)); + } + + @Test + public void parseKeyValue_emptyInput_ShouldThrowException() { + assertThrows(IllegalArgumentException.class, () -> CommandLineInputUtils.parseKeyValue(" ")); + } + + @Test + public void parseKeyValue_missingEquals_ShouldThrowException() { + assertThrows( + IllegalArgumentException.class, () -> CommandLineInputUtils.parseKeyValue("keyvalue")); + } + + @Test + public void parseKeyValue_emptyKey_ShouldThrowException() { + assertThrows( + IllegalArgumentException.class, () -> CommandLineInputUtils.parseKeyValue(" =value")); + } + + @Test + public void parseKeyValue_emptyValue_ShouldThrowException() { + assertThrows( + IllegalArgumentException.class, () -> CommandLineInputUtils.parseKeyValue("key= ")); + } + + @Test + public void parseKeyValue_multipleEquals_ShouldParseFirstOnly() { + Map.Entry result = CommandLineInputUtils.parseKeyValue("key=val=ue"); + + assertEquals("key", result.getKey()); + assertEquals("val=ue", result.getValue()); + } + + @Test + void splitByDelimiter_validSplit_shouldReturnArray() { + String[] result = CommandLineInputUtils.splitByDelimiter("a=b", "=", 2); + assertArrayEquals(new String[] {"a", "b"}, result); + } + + @Test + void splitByDelimiter_multipleDelimiters_shouldSplitAll() { + String[] result = CommandLineInputUtils.splitByDelimiter("a=b=c", "=", 0); + assertArrayEquals(new String[] {"a", "b", "c"}, result); + } + + @Test + void splitByDelimiter_nullValue_shouldThrowException() { + NullPointerException exception = + assertThrows( + NullPointerException.class, () -> CommandLineInputUtils.splitByDelimiter(null, "=", 2)); + assertTrue( + exception + .getMessage() + .contains(CoreError.DATA_LOADER_SPLIT_INPUT_VALUE_NULL.buildMessage())); + } + + @Test + void splitByDelimiter_nullDelimiter_shouldThrowException() { + NullPointerException exception = + assertThrows( + NullPointerException.class, + () -> CommandLineInputUtils.splitByDelimiter("a=b", null, 2)); + assertTrue( + exception + .getMessage() + .contains(CoreError.DATA_LOADER_SPLIT_INPUT_DELIMITER_NULL.buildMessage())); + } +} diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/DirectoryUtilsTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/DirectoryUtilsTest.java index ad4db52f0d..1ff380c19c 100755 --- a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/DirectoryUtilsTest.java +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/DirectoryUtilsTest.java @@ -18,7 +18,7 @@ /** This class tests the DirectoryValidationUtil class. */ class DirectoryUtilsTest { - private static final Logger LOGGER = LoggerFactory.getLogger(DirectoryUtilsTest.class); + private static final Logger logger = LoggerFactory.getLogger(DirectoryUtilsTest.class); @TempDir Path tempDir; @@ -81,7 +81,7 @@ private void deleteFile(Path file) { try { Files.deleteIfExists(file); } catch (IOException e) { - LOGGER.error("Failed to delete file: {}", file, e); + logger.error("Failed to delete file: {}", file, e); } } } diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/KeyUtils.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/KeyUtils.java index e3433a31b5..f20e013052 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/KeyUtils.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/KeyUtils.java @@ -213,4 +213,27 @@ private static Optional createKeyFromSource( return Optional.empty(); } } + + /** + * Convert a list of ColumnKeyValue objects to a ScalarDB Key instance. + * + * @param keyValues A list of ColumnKeyValue objects, where each object contains a column name and + * its corresponding value + * @param tableMetadata Metadata for one ScalarDB table + * @return A new ScalarDB Key instance formatted by data type + * @throws ColumnParsingException if there is an error parsing the column + */ + public static Key parseMultipleKeyValues( + List keyValues, TableMetadata tableMetadata) throws ColumnParsingException { + Key.Builder builder = Key.newBuilder(); + for (ColumnKeyValue keyValue : keyValues) { + String columnName = keyValue.getColumnName(); + String value = keyValue.getColumnValue(); + DataType columnDataType = tableMetadata.getColumnDataType(columnName); + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + Column keyValueCol = ColumnUtils.createColumnFromValue(columnDataType, columnInfo, value); + builder.add(keyValueCol); + } + return builder.build(); + } } diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/KeyUtilsTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/KeyUtilsTest.java index eb19b12c85..9379349eb4 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/KeyUtilsTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/KeyUtilsTest.java @@ -11,6 +11,7 @@ import com.scalar.db.dataloader.core.ColumnInfo; import com.scalar.db.dataloader.core.ColumnKeyValue; import com.scalar.db.dataloader.core.UnitTestUtils; +import com.scalar.db.dataloader.core.exception.ColumnParsingException; import com.scalar.db.dataloader.core.exception.KeyParsingException; import com.scalar.db.io.BigIntColumn; import com.scalar.db.io.BlobColumn; @@ -22,9 +23,11 @@ import com.scalar.db.io.Key; import com.scalar.db.io.TextColumn; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Base64; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -229,4 +232,20 @@ void createPartitionKeyFromSource_withValidData_shouldReturnValidKey() { "Optional[Key{BigIntColumn{name=col1, value=9007199254740992, hasNullValue=false}}]", key.toString()); } + + @Test + void parseMultipleKeyValues_withValidColumns_ShouldReturnValidKey() + throws ColumnParsingException { + String c1 = UnitTestUtils.TEST_COLUMN_2_CK; + ColumnKeyValue k1 = new ColumnKeyValue(c1, "1"); + String c2 = UnitTestUtils.TEST_COLUMN_3_CK; + ColumnKeyValue k2 = new ColumnKeyValue(c2, "false"); + List columnKeyValueList = new ArrayList<>(); + columnKeyValueList.add(k1); + columnKeyValueList.add(k2); + Key key = + KeyUtils.parseMultipleKeyValues( + columnKeyValueList, UnitTestUtils.createTestTableMetadata()); + assertEquals(c1, key.getColumnName(0)); + } }