Skip to content

Commit 227ec48

Browse files
feeblefakieinv-jishnuypeckstadt
authored
Backport to branch(3.13) : Add export options validator (#2455)
Co-authored-by: inv-jishnu <[email protected]> Co-authored-by: Peckstadt Yves <[email protected]>
1 parent ab373e2 commit 227ec48

File tree

4 files changed

+381
-0
lines changed

4 files changed

+381
-0
lines changed

‎core/src/main/java/com/scalar/db/common/error/CoreError.java

+28
Original file line numberDiff line numberDiff line change
@@ -678,6 +678,34 @@ public enum CoreError implements ScalarDbError {
678678
""),
679679
DATA_LOADER_ERROR_METHOD_NULL_ARGUMENT(
680680
Category.USER_ERROR, "0151", "Method null argument not allowed", "", ""),
681+
ABAC_NOT_ENABLED(
682+
Category.USER_ERROR,
683+
"0152",
684+
"The attribute-based access control feature is not enabled. To use this feature, you must enable it. Note that this feature is supported only in the ScalarDB Enterprise edition",
685+
"",
686+
""),
687+
DATA_LOADER_CLUSTERING_KEY_NOT_FOUND(
688+
Category.USER_ERROR, "0153", "The provided clustering key %s was not found", "", ""),
689+
DATA_LOADER_INVALID_PROJECTION(
690+
Category.USER_ERROR, "0154", "The column '%s' was not found", "", ""),
691+
DATA_LOADER_INCOMPLETE_PARTITION_KEY(
692+
Category.USER_ERROR,
693+
"0155",
694+
"The provided partition key is incomplete. Required key: %s",
695+
"",
696+
""),
697+
DATA_LOADER_CLUSTERING_KEY_ORDER_MISMATCH(
698+
Category.USER_ERROR,
699+
"0156",
700+
"The provided clustering key order does not match the table schema. Required order: %s",
701+
"",
702+
""),
703+
DATA_LOADER_PARTITION_KEY_ORDER_MISMATCH(
704+
Category.USER_ERROR,
705+
"0157",
706+
"The provided partition key order does not match the table schema. Required order: %s",
707+
"",
708+
""),
681709

682710
//
683711
// Errors for the concurrency error category

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidationException.java

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
/** A custom exception for export options validation errors */
4+
public class ExportOptionsValidationException extends Exception {
5+
6+
/**
7+
* Class constructor
8+
*
9+
* @param message error message
10+
*/
11+
public ExportOptionsValidationException(String message) {
12+
super(message);
13+
}
14+
}

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidator.java

+156
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
import com.scalar.db.api.Scan;
4+
import com.scalar.db.api.TableMetadata;
5+
import com.scalar.db.common.error.CoreError;
6+
import com.scalar.db.dataloader.core.ScanRange;
7+
import com.scalar.db.dataloader.core.dataexport.ExportOptions;
8+
import com.scalar.db.io.Column;
9+
import com.scalar.db.io.Key;
10+
import java.util.Iterator;
11+
import java.util.LinkedHashSet;
12+
import java.util.List;
13+
import lombok.AccessLevel;
14+
import lombok.NoArgsConstructor;
15+
16+
/**
17+
* A validator for ensuring that export options are consistent with the ScalarDB table metadata and
18+
* follow the defined constraints.
19+
*/
20+
@NoArgsConstructor(access = AccessLevel.PRIVATE)
21+
public class ExportOptionsValidator {
22+
23+
/**
24+
* Validates the export request.
25+
*
26+
* @param exportOptions The export options provided by the user.
27+
* @param tableMetadata The metadata of the ScalarDB table to validate against.
28+
* @throws ExportOptionsValidationException If the export options are invalid.
29+
*/
30+
public static void validate(ExportOptions exportOptions, TableMetadata tableMetadata)
31+
throws ExportOptionsValidationException {
32+
LinkedHashSet<String> partitionKeyNames = tableMetadata.getPartitionKeyNames();
33+
LinkedHashSet<String> clusteringKeyNames = tableMetadata.getClusteringKeyNames();
34+
ScanRange scanRange = exportOptions.getScanRange();
35+
36+
validatePartitionKey(partitionKeyNames, exportOptions.getScanPartitionKey());
37+
validateProjectionColumns(tableMetadata.getColumnNames(), exportOptions.getProjectionColumns());
38+
validateSortOrders(clusteringKeyNames, exportOptions.getSortOrders());
39+
40+
if (scanRange.getScanStartKey() != null) {
41+
validateClusteringKey(clusteringKeyNames, scanRange.getScanStartKey());
42+
}
43+
if (scanRange.getScanEndKey() != null) {
44+
validateClusteringKey(clusteringKeyNames, scanRange.getScanEndKey());
45+
}
46+
}
47+
48+
/*
49+
* Check if the provided partition key is available in the ScalarDB table
50+
* @param partitionKeyNames List of partition key names available in a
51+
* @param key To be validated ScalarDB key
52+
* @throws ExportOptionsValidationException if the key could not be found or is not a partition
53+
*/
54+
private static void validatePartitionKey(LinkedHashSet<String> partitionKeyNames, Key key)
55+
throws ExportOptionsValidationException {
56+
if (partitionKeyNames == null || key == null) {
57+
return;
58+
}
59+
60+
// Make sure that all partition key columns are provided
61+
if (partitionKeyNames.size() != key.getColumns().size()) {
62+
throw new ExportOptionsValidationException(
63+
CoreError.DATA_LOADER_INCOMPLETE_PARTITION_KEY.buildMessage(partitionKeyNames));
64+
}
65+
66+
// Check if the order of columns in key.getColumns() matches the order in partitionKeyNames
67+
Iterator<String> partitionKeyIterator = partitionKeyNames.iterator();
68+
for (Column<?> column : key.getColumns()) {
69+
// Check if the column names match in order
70+
if (!partitionKeyIterator.hasNext()
71+
|| !partitionKeyIterator.next().equals(column.getName())) {
72+
throw new ExportOptionsValidationException(
73+
CoreError.DATA_LOADER_PARTITION_KEY_ORDER_MISMATCH.buildMessage(partitionKeyNames));
74+
}
75+
}
76+
}
77+
78+
private static void validateSortOrders(
79+
LinkedHashSet<String> clusteringKeyNames, List<Scan.Ordering> sortOrders)
80+
throws ExportOptionsValidationException {
81+
if (sortOrders == null || sortOrders.isEmpty()) {
82+
return;
83+
}
84+
85+
for (Scan.Ordering sortOrder : sortOrders) {
86+
checkIfColumnExistsAsClusteringKey(clusteringKeyNames, sortOrder.getColumnName());
87+
}
88+
}
89+
90+
/**
91+
* Validates that the clustering key columns in the given Key object match the expected order
92+
* defined in the clusteringKeyNames. The Key can be a prefix of the clusteringKeyNames, but the
93+
* order must remain consistent.
94+
*
95+
* @param clusteringKeyNames the expected ordered set of clustering key names
96+
* @param key the Key object containing the actual clustering key columns
97+
* @throws ExportOptionsValidationException if the order or names of clustering keys do not match
98+
*/
99+
private static void validateClusteringKey(LinkedHashSet<String> clusteringKeyNames, Key key)
100+
throws ExportOptionsValidationException {
101+
// If either clusteringKeyNames or key is null, no validation is needed
102+
if (clusteringKeyNames == null || key == null) {
103+
return;
104+
}
105+
106+
// Create an iterator to traverse the clusteringKeyNames in order
107+
Iterator<String> clusteringKeyIterator = clusteringKeyNames.iterator();
108+
109+
// Iterate through the columns in the given Key
110+
for (Column<?> column : key.getColumns()) {
111+
// If clusteringKeyNames have been exhausted but columns still exist in the Key,
112+
// it indicates a mismatch
113+
if (!clusteringKeyIterator.hasNext()) {
114+
throw new ExportOptionsValidationException(
115+
CoreError.DATA_LOADER_CLUSTERING_KEY_ORDER_MISMATCH.buildMessage(clusteringKeyNames));
116+
}
117+
118+
// Get the next expected clustering key name
119+
String expectedKey = clusteringKeyIterator.next();
120+
121+
// Check if the current column name matches the expected clustering key name
122+
if (!column.getName().equals(expectedKey)) {
123+
throw new ExportOptionsValidationException(
124+
CoreError.DATA_LOADER_CLUSTERING_KEY_ORDER_MISMATCH.buildMessage(clusteringKeyNames));
125+
}
126+
}
127+
}
128+
129+
private static void checkIfColumnExistsAsClusteringKey(
130+
LinkedHashSet<String> clusteringKeyNames, String columnName)
131+
throws ExportOptionsValidationException {
132+
if (clusteringKeyNames == null || columnName == null) {
133+
return;
134+
}
135+
136+
if (!clusteringKeyNames.contains(columnName)) {
137+
throw new ExportOptionsValidationException(
138+
CoreError.DATA_LOADER_CLUSTERING_KEY_NOT_FOUND.buildMessage(columnName));
139+
}
140+
}
141+
142+
private static void validateProjectionColumns(
143+
LinkedHashSet<String> columnNames, List<String> columns)
144+
throws ExportOptionsValidationException {
145+
if (columns == null || columns.isEmpty()) {
146+
return;
147+
}
148+
149+
for (String column : columns) {
150+
if (!columnNames.contains(column)) {
151+
throw new ExportOptionsValidationException(
152+
CoreError.DATA_LOADER_INVALID_PROJECTION.buildMessage(column));
153+
}
154+
}
155+
}
156+
}

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidatorTest.java

+183
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
4+
5+
import com.scalar.db.api.TableMetadata;
6+
import com.scalar.db.common.error.CoreError;
7+
import com.scalar.db.dataloader.core.FileFormat;
8+
import com.scalar.db.dataloader.core.ScanRange;
9+
import com.scalar.db.dataloader.core.dataexport.ExportOptions;
10+
import com.scalar.db.io.DataType;
11+
import com.scalar.db.io.IntColumn;
12+
import com.scalar.db.io.Key;
13+
import com.scalar.db.io.TextColumn;
14+
import java.util.ArrayList;
15+
import java.util.Collections;
16+
import java.util.LinkedHashSet;
17+
import java.util.List;
18+
import org.junit.jupiter.api.BeforeEach;
19+
import org.junit.jupiter.api.Test;
20+
21+
class ExportOptionsValidatorTest {
22+
23+
private TableMetadata singlePkCkMetadata;
24+
private TableMetadata multiplePkCkMetadata;
25+
private List<String> projectedColumns;
26+
27+
@BeforeEach
28+
void setup() {
29+
singlePkCkMetadata = createMockMetadata(1, 1);
30+
multiplePkCkMetadata = createMockMetadata(2, 2);
31+
projectedColumns = createProjectedColumns();
32+
}
33+
34+
private TableMetadata createMockMetadata(int pkCount, int ckCount) {
35+
TableMetadata.Builder builder = TableMetadata.newBuilder();
36+
37+
// Add partition keys
38+
for (int i = 1; i <= pkCount; i++) {
39+
builder.addColumn("pk" + i, DataType.INT);
40+
builder.addPartitionKey("pk" + i);
41+
}
42+
43+
// Add clustering keys
44+
for (int i = 1; i <= ckCount; i++) {
45+
builder.addColumn("ck" + i, DataType.TEXT);
46+
builder.addClusteringKey("ck" + i);
47+
}
48+
49+
return builder.build();
50+
}
51+
52+
private List<String> createProjectedColumns() {
53+
List<String> columns = new ArrayList<>();
54+
columns.add("pk1");
55+
columns.add("ck1");
56+
return columns;
57+
}
58+
59+
@Test
60+
void validate_withValidExportOptionsForSinglePkCk_ShouldNotThrowException()
61+
throws ExportOptionsValidationException {
62+
63+
Key partitionKey = Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
64+
65+
ExportOptions exportOptions =
66+
ExportOptions.builder("test", "sample", partitionKey, FileFormat.JSON)
67+
.projectionColumns(projectedColumns)
68+
.scanRange(new ScanRange(null, null, false, false))
69+
.build();
70+
71+
ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata);
72+
}
73+
74+
@Test
75+
void validate_withValidExportOptionsForMultiplePkCk_ShouldNotThrowException()
76+
throws ExportOptionsValidationException {
77+
78+
Key partitionKey =
79+
Key.newBuilder().add(IntColumn.of("pk1", 1)).add(IntColumn.of("pk2", 2)).build();
80+
81+
ExportOptions exportOptions =
82+
ExportOptions.builder("test", "sample", partitionKey, FileFormat.JSON)
83+
.projectionColumns(projectedColumns)
84+
.scanRange(new ScanRange(null, null, false, false))
85+
.build();
86+
87+
ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata);
88+
}
89+
90+
@Test
91+
void validate_withIncompletePartitionKeyForSinglePk_ShouldThrowException() {
92+
Key incompletePartitionKey = Key.newBuilder().build();
93+
94+
ExportOptions exportOptions =
95+
ExportOptions.builder("test", "sample", incompletePartitionKey, FileFormat.JSON).build();
96+
97+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
98+
.isInstanceOf(ExportOptionsValidationException.class)
99+
.hasMessage(
100+
CoreError.DATA_LOADER_INCOMPLETE_PARTITION_KEY.buildMessage(
101+
singlePkCkMetadata.getPartitionKeyNames()));
102+
}
103+
104+
@Test
105+
void validate_withIncompletePartitionKeyForMultiplePks_ShouldThrowException() {
106+
Key incompletePartitionKey = Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
107+
108+
ExportOptions exportOptions =
109+
ExportOptions.builder("test", "sample", incompletePartitionKey, FileFormat.JSON).build();
110+
111+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata))
112+
.isInstanceOf(ExportOptionsValidationException.class)
113+
.hasMessage(
114+
CoreError.DATA_LOADER_INCOMPLETE_PARTITION_KEY.buildMessage(
115+
multiplePkCkMetadata.getPartitionKeyNames()));
116+
}
117+
118+
@Test
119+
void validate_withInvalidProjectionColumn_ShouldThrowException() {
120+
ExportOptions exportOptions =
121+
ExportOptions.builder(
122+
"test",
123+
"sample",
124+
Key.newBuilder().add(IntColumn.of("pk1", 1)).build(),
125+
FileFormat.JSON)
126+
.projectionColumns(Collections.singletonList("invalid_column"))
127+
.build();
128+
129+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
130+
.isInstanceOf(ExportOptionsValidationException.class)
131+
.hasMessage(CoreError.DATA_LOADER_INVALID_PROJECTION.buildMessage("invalid_column"));
132+
}
133+
134+
@Test
135+
void validate_withInvalidClusteringKeyInScanRange_ShouldThrowException() {
136+
ScanRange scanRange =
137+
new ScanRange(
138+
Key.newBuilder().add(TextColumn.of("invalid_ck", "value")).build(),
139+
Key.newBuilder().add(TextColumn.of("ck1", "value")).build(),
140+
false,
141+
false);
142+
143+
ExportOptions exportOptions =
144+
ExportOptions.builder("test", "sample", createValidPartitionKey(), FileFormat.JSON)
145+
.scanRange(scanRange)
146+
.build();
147+
148+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
149+
.isInstanceOf(ExportOptionsValidationException.class)
150+
.hasMessage(CoreError.DATA_LOADER_CLUSTERING_KEY_ORDER_MISMATCH.buildMessage("[ck1]"));
151+
}
152+
153+
@Test
154+
void validate_withInvalidPartitionKeyOrder_ShouldThrowException() {
155+
// Partition key names are expected to be "pk1", "pk2"
156+
LinkedHashSet<String> partitionKeyNames = new LinkedHashSet<>();
157+
partitionKeyNames.add("pk1");
158+
partitionKeyNames.add("pk2");
159+
160+
// Create a partition key with reversed order, expecting an error
161+
Key invalidPartitionKey =
162+
Key.newBuilder()
163+
.add(IntColumn.of("pk2", 2)) // Incorrect order
164+
.add(IntColumn.of("pk1", 1)) // Incorrect order
165+
.build();
166+
167+
ExportOptions exportOptions =
168+
ExportOptions.builder("test", "sample", invalidPartitionKey, FileFormat.JSON)
169+
.projectionColumns(projectedColumns)
170+
.scanRange(new ScanRange(null, null, false, false))
171+
.build();
172+
173+
// Verify that the validator throws the correct exception
174+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata))
175+
.isInstanceOf(ExportOptionsValidationException.class)
176+
.hasMessage(
177+
CoreError.DATA_LOADER_PARTITION_KEY_ORDER_MISMATCH.buildMessage(partitionKeyNames));
178+
}
179+
180+
private Key createValidPartitionKey() {
181+
return Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
182+
}
183+
}

0 commit comments

Comments
 (0)