Skip to content

Commit 6282ce7

Browse files
committed
Retry flaky unified tests
JAVA-5393
1 parent 90976e4 commit 6282ce7

File tree

4 files changed

+76
-28
lines changed

4 files changed

+76
-28
lines changed

driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java

+2
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ public void shouldPassAllOutcomes(
102102
@Nullable final String fileDescription,
103103
@Nullable final String testDescription,
104104
@Nullable final String directoryName,
105+
final int attemptNumber,
105106
final String schemaVersion,
106107
@Nullable final BsonArray runOnRequirements,
107108
final BsonArray entitiesArray,
@@ -111,6 +112,7 @@ public void shouldPassAllOutcomes(
111112
super.shouldPassAllOutcomes(fileDescription,
112113
testDescription,
113114
directoryName,
115+
attemptNumber,
114116
schemaVersion,
115117
runOnRequirements,
116118
entitiesArray,

driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java

+62-27
Original file line numberDiff line numberDiff line change
@@ -21,27 +21,27 @@
2121
import com.mongodb.MongoNamespace;
2222
import com.mongodb.ReadPreference;
2323
import com.mongodb.UnixServerAddress;
24-
import com.mongodb.client.unified.UnifiedTestModifications.TestDef;
25-
import com.mongodb.event.TestServerMonitorListener;
26-
import com.mongodb.internal.logging.LogMessage;
27-
import com.mongodb.logging.TestLoggingInterceptor;
2824
import com.mongodb.WriteConcern;
2925
import com.mongodb.client.ClientSession;
3026
import com.mongodb.client.MongoClient;
3127
import com.mongodb.client.MongoDatabase;
3228
import com.mongodb.client.gridfs.GridFSBucket;
3329
import com.mongodb.client.model.Filters;
3430
import com.mongodb.client.test.CollectionHelper;
31+
import com.mongodb.client.unified.UnifiedTestModifications.TestDef;
3532
import com.mongodb.client.vault.ClientEncryption;
3633
import com.mongodb.connection.ClusterDescription;
3734
import com.mongodb.connection.ClusterType;
3835
import com.mongodb.connection.ServerDescription;
3936
import com.mongodb.event.CommandEvent;
4037
import com.mongodb.event.CommandStartedEvent;
38+
import com.mongodb.event.TestServerMonitorListener;
4139
import com.mongodb.internal.connection.TestCommandListener;
4240
import com.mongodb.internal.connection.TestConnectionPoolListener;
41+
import com.mongodb.internal.logging.LogMessage;
4342
import com.mongodb.lang.NonNull;
4443
import com.mongodb.lang.Nullable;
44+
import com.mongodb.logging.TestLoggingInterceptor;
4545
import com.mongodb.test.AfterBeforeParameterResolver;
4646
import org.bson.BsonArray;
4747
import org.bson.BsonBoolean;
@@ -57,14 +57,17 @@
5757
import org.junit.jupiter.params.ParameterizedTest;
5858
import org.junit.jupiter.params.provider.Arguments;
5959
import org.junit.jupiter.params.provider.MethodSource;
60+
import org.opentest4j.AssertionFailedError;
6061
import org.opentest4j.TestAbortedException;
6162

6263
import java.io.File;
6364
import java.io.IOException;
6465
import java.net.URISyntaxException;
66+
import java.text.MessageFormat;
6567
import java.util.ArrayList;
6668
import java.util.Collection;
6769
import java.util.Collections;
70+
import java.util.HashSet;
6871
import java.util.List;
6972
import java.util.Set;
7073
import java.util.concurrent.ExecutionException;
@@ -91,6 +94,7 @@
9194
import static org.junit.jupiter.api.Assertions.assertNull;
9295
import static org.junit.jupiter.api.Assertions.assertTrue;
9396
import static org.junit.jupiter.api.Assertions.fail;
97+
import static org.junit.jupiter.api.Assumptions.assumeFalse;
9498
import static org.junit.jupiter.api.Assumptions.assumeTrue;
9599
import static util.JsonPoweredTestHelper.getTestDocument;
96100
import static util.JsonPoweredTestHelper.getTestFiles;
@@ -100,6 +104,9 @@ public abstract class UnifiedTest {
100104
private static final Set<String> PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton(
101105
"wait queue timeout errors include details about checked out connections");
102106

107+
public static final int ATTEMPTS = 3;
108+
private static Set<String> completed = new HashSet<>();
109+
103110
@Nullable
104111
private String fileDescription;
105112
private String schemaVersion;
@@ -153,26 +160,40 @@ public Entities getEntities() {
153160
return entities;
154161
}
155162

163+
public int attempts() {
164+
return ATTEMPTS;
165+
}
166+
156167
@NonNull
157168
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
169+
return getTestData(directory, ATTEMPTS);
170+
}
171+
172+
@NonNull
173+
protected static Collection<Arguments> getTestData(final String directory, final int attempts) throws URISyntaxException, IOException {
158174
List<Arguments> data = new ArrayList<>();
159175
for (File file : getTestFiles("/" + directory + "/")) {
160176
BsonDocument fileDocument = getTestDocument(file);
161-
162177
for (BsonValue cur : fileDocument.getArray("tests")) {
163-
data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument()));
178+
for (int attempt = 1; attempt <= attempts; attempt++) {
179+
data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument(), attempt));
180+
}
164181
}
165182
}
166183
return data;
167184
}
168185

169186
@NonNull
170187
private static Arguments createTestData(
171-
final String directory, final BsonDocument fileDocument, final BsonDocument testDocument) {
188+
final String directory,
189+
final BsonDocument fileDocument,
190+
final BsonDocument testDocument,
191+
final int attempt) {
172192
return Arguments.of(
173193
fileDocument.getString("description").getValue(),
174194
testDocument.getString("description").getValue(),
175195
directory,
196+
attempt,
176197
fileDocument.getString("schemaVersion").getValue(),
177198
fileDocument.getArray("runOnRequirements", null),
178199
fileDocument.getArray("createEntities", new BsonArray()),
@@ -195,6 +216,7 @@ public void setUp(
195216
@Nullable final String fileDescription,
196217
@Nullable final String testDescription,
197218
@Nullable final String directoryName,
219+
final int attemptNumber,
198220
final String schemaVersion,
199221
@Nullable final BsonArray runOnRequirements,
200222
final BsonArray entitiesArray,
@@ -293,40 +315,53 @@ protected boolean isReactive() {
293315
return false;
294316
}
295317

296-
@ParameterizedTest(name = "{0}: {1}")
318+
@ParameterizedTest(name = "{0}: {1} ({3})")
297319
@MethodSource("data")
298320
public void shouldPassAllOutcomes(
299321
@Nullable final String fileDescription,
300322
@Nullable final String testDescription,
301323
@Nullable final String directoryName,
324+
final int attemptNumber,
302325
final String schemaVersion,
303326
@Nullable final BsonArray runOnRequirements,
304327
final BsonArray entitiesArray,
305328
final BsonArray initialData,
306329
final BsonDocument definition) {
307-
BsonArray operations = definition.getArray("operations");
308-
for (int i = 0; i < operations.size(); i++) {
309-
BsonValue cur = operations.get(i);
310-
assertOperation(rootContext, cur.asDocument(), i);
311-
}
330+
String testId = MessageFormat.format("{0}: {1}", fileDescription, testDescription);
331+
assumeFalse(completed.contains(testId), "Skipping test already performed");
332+
completed.add(testId);
333+
try {
334+
BsonArray operations = definition.getArray("operations");
335+
for (int i = 0; i < operations.size(); i++) {
336+
BsonValue cur = operations.get(i);
337+
assertOperation(rootContext, cur.asDocument(), i);
338+
}
312339

313-
if (definition.containsKey("outcome")) {
314-
assertOutcome(rootContext);
315-
}
340+
if (definition.containsKey("outcome")) {
341+
assertOutcome(rootContext);
342+
}
316343

317-
if (definition.containsKey("expectEvents")) {
318-
compareEvents(rootContext, definition);
319-
}
344+
if (definition.containsKey("expectEvents")) {
345+
compareEvents(rootContext, definition);
346+
}
320347

321-
if (definition.containsKey("expectLogMessages")) {
322-
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
323-
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
324-
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
325-
if (getMongoClientSettings().getClusterSettings()
326-
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
327-
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
348+
if (definition.containsKey("expectLogMessages")) {
349+
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
350+
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
351+
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
352+
if (getMongoClientSettings().getClusterSettings()
353+
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
354+
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
355+
}
356+
compareLogMessages(rootContext, definition, tweaks);
357+
}
358+
} catch (AssertionFailedError e) {
359+
completed.remove(testId);
360+
if (attemptNumber == attempts()) { // last attempt
361+
throw e;
362+
} else {
363+
assumeFalse(completed.contains(testId), "Ignoring failure and retrying attempt " + attemptNumber);
328364
}
329-
compareLogMessages(rootContext, definition, tweaks);
330365
}
331366
}
332367

driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java

+10-1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public void setUp(
3939
final String directoryName,
4040
@Nullable final String fileDescription,
4141
@Nullable final String testDescription,
42+
final int attemptNumber,
4243
final String schemaVersion,
4344
@Nullable final BsonArray runOnRequirements,
4445
final BsonArray entitiesArray,
@@ -49,6 +50,7 @@ public void setUp(
4950
directoryName,
5051
fileDescription,
5152
testDescription,
53+
attemptNumber,
5254
schemaVersion,
5355
runOnRequirements,
5456
entitiesArray,
@@ -66,6 +68,7 @@ public void shouldPassAllOutcomes(
6668
@Nullable final String fileDescription,
6769
@Nullable final String testDescription,
6870
@Nullable final String directoryName,
71+
final int attemptNumber,
6972
final String schemaVersion,
7073
@Nullable final BsonArray runOnRequirements,
7174
final BsonArray entitiesArray,
@@ -77,6 +80,7 @@ public void shouldPassAllOutcomes(
7780
fileDescription,
7881
testDescription,
7982
directoryName,
83+
attemptNumber,
8084
schemaVersion,
8185
runOnRequirements,
8286
entitiesArray,
@@ -90,6 +94,11 @@ public void shouldPassAllOutcomes(
9094
}
9195

9296
private static Collection<Arguments> data() throws URISyntaxException, IOException {
93-
return getTestData("unified-test-format/valid-fail");
97+
return getTestData("unified-test-format/valid-fail", 1);
98+
}
99+
100+
@Override
101+
public int attempts() {
102+
return 1;
94103
}
95104
}

driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java

+2
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ protected boolean terminateLoop() {
101101
null,
102102
null,
103103
null,
104+
unifiedTest.attempts(),
104105
schemaVersion,
105106
runOnRequirements,
106107
createEntities,
@@ -110,6 +111,7 @@ protected boolean terminateLoop() {
110111
null,
111112
null,
112113
null,
114+
unifiedTest.attempts(),
113115
schemaVersion,
114116
runOnRequirements,
115117
createEntities,

0 commit comments

Comments
 (0)