Skip to content

Commit 8a6116d

Browse files
authored
feat: add configurable batch size infrastructure (#188)
* fix(PlatformClient): validate userAgents is not null * feat(DocumentUploadQueue): add configurable batch size infrastructure - Add constants: MAX_ALLOWED_QUEUE_SIZE (256MB), DEFAULT_QUEUE_SIZE (256MB) - Add BATCH_SIZE_PROPERTY system property for runtime configuration - Add getConfiguredBatchSize() static method for system property lookup - Add new constructor with maxQueueSize parameter - Add validation to prevent exceeding 256MB API limit - Update existing constructor to use getConfiguredBatchSize() as default Configuration priority: constructor param > system property > default (256MB) * remove sisyphus * format * run spotless
1 parent 45a3917 commit 8a6116d

File tree

2 files changed

+108
-15
lines changed

2 files changed

+108
-15
lines changed

src/main/java/com/coveo/pushapiclient/DocumentUploadQueue.java

Lines changed: 100 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,118 @@
88
/** Represents a queue for uploading documents using a specified upload strategy */
99
class DocumentUploadQueue {
1010
private static final Logger logger = LogManager.getLogger(DocumentUploadQueue.class);
11-
protected final UploadStrategy uploader;
12-
protected final int maxQueueSize = 5 * 1024 * 1024;
11+
12+
/** Maximum allowed queue size based on Stream API limit (256 MB) */
13+
protected static final int MAX_ALLOWED_QUEUE_SIZE = 256 * 1024 * 1024;
14+
15+
/** Default queue size (5 MB) */
16+
protected static final int DEFAULT_QUEUE_SIZE = 5 * 1024 * 1024;
17+
18+
/** System property name for configuring the default batch size */
19+
public static final String BATCH_SIZE_PROPERTY = "coveo.push.batchSize";
20+
21+
protected UploadStrategy uploader;
22+
protected final int maxQueueSize;
1323
protected ArrayList<DocumentBuilder> documentToAddList;
1424
protected ArrayList<DeleteDocument> documentToDeleteList;
1525
protected int size;
1626

1727
/**
18-
* Constructs a new DocumentUploadQueue object with a default maximum queue size limit of 5MB.
28+
* Validates batch size against constraints (> 0 and <= 256MB). Used by getConfiguredBatchSize and
29+
* constructors to ensure consistent validation logic.
30+
*
31+
* @param sizeBytes The batch size in bytes to validate
32+
* @throws IllegalArgumentException if size exceeds MAX_ALLOWED_QUEUE_SIZE or is <= 0
33+
*/
34+
protected static void validateBatchSize(int sizeBytes) {
35+
if (sizeBytes > MAX_ALLOWED_QUEUE_SIZE) {
36+
throw new IllegalArgumentException(
37+
String.format(
38+
"Batch size (%d bytes) exceeds the Stream API limit of %d bytes (%d MB)",
39+
sizeBytes, MAX_ALLOWED_QUEUE_SIZE, MAX_ALLOWED_QUEUE_SIZE / (1024 * 1024)));
40+
}
41+
if (sizeBytes <= 0) {
42+
throw new IllegalArgumentException("Batch size must be greater than 0");
43+
}
44+
}
45+
46+
/**
47+
* Gets the configured batch size from system properties, or returns the default if not set.
48+
*
49+
* <p>The system property is read as bytes. When not set, returns DEFAULT_QUEUE_SIZE (5 MB).
50+
*
51+
* <p>Example: Set a 50 MB batch size via system property:
52+
*
53+
* <pre>
54+
* java -Dcoveo.push.batchSize=52428800 -jar app.jar // 50 * 1024 * 1024 bytes
55+
* </pre>
56+
*
57+
* @return The configured batch size in bytes (e.g., 52428800 for 50 MB)
58+
* @throws IllegalArgumentException if the configured value exceeds 256MB or is invalid
59+
*/
60+
public static int getConfiguredBatchSize() {
61+
String propertyValue = System.getProperty(BATCH_SIZE_PROPERTY);
62+
if (propertyValue == null || propertyValue.trim().isEmpty()) {
63+
return DEFAULT_QUEUE_SIZE;
64+
}
65+
66+
int configuredSize;
67+
try {
68+
configuredSize = Integer.parseInt(propertyValue.trim());
69+
} catch (NumberFormatException e) {
70+
throw new IllegalArgumentException(
71+
String.format(
72+
"Invalid value for system property %s: '%s'. Must be a valid integer.",
73+
BATCH_SIZE_PROPERTY, propertyValue),
74+
e);
75+
}
76+
77+
validateBatchSize(configuredSize);
78+
79+
logger.info(
80+
String.format(
81+
"Using configured batch size from system property %s: %d bytes (%.2f MB)",
82+
BATCH_SIZE_PROPERTY, configuredSize, configuredSize / (1024.0 * 1024.0)));
83+
return configuredSize;
84+
}
85+
86+
/**
87+
* Constructs a new DocumentUploadQueue with the default batch size.
88+
*
89+
* <p>Uses the configured batch size from system property "coveo.push.batchSize" if set, otherwise
90+
* defaults to DEFAULT_QUEUE_SIZE (5 MB = 5242880 bytes).
1991
*
2092
* @param uploader The upload strategy to be used for document uploads.
93+
* @throws IllegalArgumentException if the system property value exceeds 256MB or is invalid.
2194
*/
2295
public DocumentUploadQueue(UploadStrategy uploader) {
96+
this(uploader, getConfiguredBatchSize());
97+
}
98+
99+
/**
100+
* Constructs a new DocumentUploadQueue object with a configurable maximum queue size limit.
101+
*
102+
* @param uploader The upload strategy to be used for document uploads.
103+
* @param maxQueueSize The maximum queue size in bytes (e.g., 52428800 for 50 MB). Must not exceed
104+
* 256MB (Stream API limit).
105+
* @throws IllegalArgumentException if maxQueueSize exceeds the API limit of 256MB.
106+
*/
107+
public DocumentUploadQueue(UploadStrategy uploader, int maxQueueSize) {
108+
validateBatchSize(maxQueueSize);
23109
this.documentToAddList = new ArrayList<>();
24110
this.documentToDeleteList = new ArrayList<>();
25111
this.uploader = uploader;
112+
this.maxQueueSize = maxQueueSize;
113+
}
114+
115+
/**
116+
* Default constructor for testing purposes (used by Mockito @InjectMocks). Initializes with
117+
* default batch size; uploader is injected by Mockito.
118+
*/
119+
public DocumentUploadQueue() {
120+
this.documentToAddList = new ArrayList<>();
121+
this.documentToDeleteList = new ArrayList<>();
122+
this.maxQueueSize = DEFAULT_QUEUE_SIZE;
26123
}
27124

28125
/**

src/test/java/com/coveo/pushapiclient/DocumentUploadQueueTest.java

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,37 +12,31 @@
1212
import org.junit.After;
1313
import org.junit.Before;
1414
import org.junit.Test;
15-
import org.mockito.InjectMocks;
1615
import org.mockito.Mock;
1716
import org.mockito.MockitoAnnotations;
1817

1918
public class DocumentUploadQueueTest {
2019

21-
@Mock private UploadStrategy uploadStrategy;
20+
private static final int TEST_BATCH_SIZE = 5 * 1024 * 1024;
2221

23-
@InjectMocks private DocumentUploadQueue queue;
22+
@Mock private UploadStrategy uploadStrategy;
2423

24+
private DocumentUploadQueue queue;
2525
private AutoCloseable closeable;
2626
private DocumentBuilder documentToAdd;
2727
private DeleteDocument documentToDelete;
2828

2929
private int oneMegaByte = 1 * 1024 * 1024;
3030

3131
private String generateStringFromBytes(int numBytes) {
32-
// Check if the number of bytes is valid
3332
if (numBytes <= 0) {
3433
return "";
3534
}
36-
37-
// Create a byte array with the specified length
3835
byte[] bytes = new byte[numBytes];
39-
40-
// Fill the byte array with a pattern of ASCII characters
41-
byte pattern = 65; // ASCII value for 'A'
36+
byte pattern = 65;
4237
for (int i = 0; i < numBytes; i++) {
4338
bytes[i] = pattern;
4439
}
45-
4640
return new String(bytes);
4741
}
4842

@@ -53,15 +47,17 @@ private DocumentBuilder generateDocumentFromSize(int numBytes) {
5347

5448
@Before
5549
public void setup() {
50+
closeable = MockitoAnnotations.openMocks(this);
51+
52+
queue = new DocumentUploadQueue(uploadStrategy, TEST_BATCH_SIZE);
53+
5654
String twoMegaByteData = generateStringFromBytes(2 * oneMegaByte);
5755

5856
documentToAdd =
5957
new DocumentBuilder("https://my.document.uri?ref=1", "My new document")
6058
.withData(twoMegaByteData);
6159

6260
documentToDelete = new DeleteDocument("https://my.document.uri?ref=3");
63-
64-
closeable = MockitoAnnotations.openMocks(this);
6561
}
6662

6763
@After

0 commit comments

Comments
 (0)