@@ -17,10 +17,14 @@ const {
17
17
TEST_SOURCE_START ,
18
18
TEST_TYPE ,
19
19
TEST_SOURCE_FILE ,
20
- TEST_CONFIGURATION_BROWSER_NAME
20
+ TEST_CONFIGURATION_BROWSER_NAME ,
21
+ TEST_IS_NEW ,
22
+ TEST_IS_RETRY
21
23
} = require ( '../../packages/dd-trace/src/plugins/util/test' )
22
24
const { ERROR_MESSAGE } = require ( '../../packages/dd-trace/src/constants' )
23
25
26
+ const NUM_RETRIES_EFD = 3
27
+
24
28
const versions = [ '1.18.0' , 'latest' ]
25
29
26
30
versions . forEach ( ( version ) => {
@@ -217,5 +221,218 @@ versions.forEach((version) => {
217
221
}
218
222
)
219
223
} )
224
+
225
+ if ( version === 'latest' ) {
226
+ context ( 'early flake detection' , ( ) => {
227
+ it ( 'retries new tests' , ( done ) => {
228
+ receiver . setSettings ( {
229
+ itr_enabled : false ,
230
+ code_coverage : false ,
231
+ tests_skipping : false ,
232
+ early_flake_detection : {
233
+ enabled : true ,
234
+ slow_test_retries : {
235
+ '5s' : NUM_RETRIES_EFD
236
+ }
237
+ }
238
+ } )
239
+
240
+ receiver . setKnownTests (
241
+ {
242
+ playwright : {
243
+ 'landing-page-test.js' : [
244
+ // 'should work with passing tests', // it will be considered new
245
+ 'should work with skipped tests' ,
246
+ 'should work with fixme' ,
247
+ 'should work with annotated tests'
248
+ ] ,
249
+ 'skipped-suite-test.js' : [
250
+ 'should work with fixme root'
251
+ ] ,
252
+ 'todo-list-page-test.js' : [
253
+ 'should work with failing tests' ,
254
+ 'should work with fixme root'
255
+ ]
256
+ }
257
+ }
258
+ )
259
+
260
+ const receiverPromise = receiver
261
+ . gatherPayloadsMaxTimeout ( ( { url } ) => url === '/api/v2/citestcycle' , ( payloads ) => {
262
+ const events = payloads . flatMap ( ( { payload } ) => payload . events )
263
+ const tests = events . filter ( event => event . type === 'test' ) . map ( event => event . content )
264
+
265
+ const newTests = tests . filter ( test =>
266
+ test . resource ===
267
+ 'landing-page-test.js.should work with passing tests'
268
+ )
269
+ newTests . forEach ( test => {
270
+ assert . propertyVal ( test . meta , TEST_IS_NEW , 'true' )
271
+ } )
272
+
273
+ const retriedTests = tests . filter ( test => test . meta [ TEST_IS_RETRY ] === 'true' )
274
+
275
+ assert . equal ( retriedTests . length , NUM_RETRIES_EFD )
276
+
277
+ // all but one has been retried
278
+ assert . equal ( retriedTests . length , newTests . length - 1 )
279
+ } )
280
+
281
+ childProcess = exec (
282
+ './node_modules/.bin/playwright test -c playwright.config.js' ,
283
+ {
284
+ cwd,
285
+ env : {
286
+ ...getCiVisAgentlessConfig ( receiver . port ) ,
287
+ PW_BASE_URL : `http://localhost:${ webAppPort } `
288
+ } ,
289
+ stdio : 'pipe'
290
+ }
291
+ )
292
+
293
+ childProcess . on ( 'exit' , ( ) => {
294
+ receiverPromise . then ( ( ) => done ( ) ) . catch ( done )
295
+ } )
296
+ } )
297
+ it ( 'is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false' , ( done ) => {
298
+ receiver . setSettings ( {
299
+ itr_enabled : false ,
300
+ code_coverage : false ,
301
+ tests_skipping : false ,
302
+ early_flake_detection : {
303
+ enabled : true ,
304
+ slow_test_retries : {
305
+ '5s' : NUM_RETRIES_EFD
306
+ }
307
+ }
308
+ } )
309
+
310
+ receiver . setKnownTests (
311
+ {
312
+ playwright : {
313
+ 'landing-page-test.js' : [
314
+ // 'should work with passing tests', // it will be considered new
315
+ 'should work with skipped tests' ,
316
+ 'should work with fixme' ,
317
+ 'should work with annotated tests'
318
+ ] ,
319
+ 'skipped-suite-test.js' : [
320
+ 'should work with fixme root'
321
+ ] ,
322
+ 'todo-list-page-test.js' : [
323
+ 'should work with failing tests' ,
324
+ 'should work with fixme root'
325
+ ]
326
+ }
327
+ }
328
+ )
329
+
330
+ const receiverPromise = receiver
331
+ . gatherPayloadsMaxTimeout ( ( { url } ) => url === '/api/v2/citestcycle' , ( payloads ) => {
332
+ const events = payloads . flatMap ( ( { payload } ) => payload . events )
333
+ const tests = events . filter ( event => event . type === 'test' ) . map ( event => event . content )
334
+
335
+ const newTests = tests . filter ( test =>
336
+ test . resource ===
337
+ 'landing-page-test.js.should work with passing tests'
338
+ )
339
+ newTests . forEach ( test => {
340
+ assert . notProperty ( test . meta , TEST_IS_NEW )
341
+ } )
342
+
343
+ const retriedTests = tests . filter ( test => test . meta [ TEST_IS_RETRY ] === 'true' )
344
+
345
+ assert . equal ( retriedTests . length , 0 )
346
+ } )
347
+
348
+ childProcess = exec (
349
+ './node_modules/.bin/playwright test -c playwright.config.js' ,
350
+ {
351
+ cwd,
352
+ env : {
353
+ ...getCiVisAgentlessConfig ( receiver . port ) ,
354
+ PW_BASE_URL : `http://localhost:${ webAppPort } ` ,
355
+ DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED : 'false'
356
+ } ,
357
+ stdio : 'pipe'
358
+ }
359
+ )
360
+
361
+ childProcess . on ( 'exit' , ( ) => {
362
+ receiverPromise . then ( ( ) => done ( ) ) . catch ( done )
363
+ } )
364
+ } )
365
+ it ( 'does not retry tests that are skipped' , ( done ) => {
366
+ receiver . setSettings ( {
367
+ itr_enabled : false ,
368
+ code_coverage : false ,
369
+ tests_skipping : false ,
370
+ early_flake_detection : {
371
+ enabled : true ,
372
+ slow_test_retries : {
373
+ '5s' : NUM_RETRIES_EFD
374
+ }
375
+ }
376
+ } )
377
+
378
+ receiver . setKnownTests (
379
+ {
380
+ playwright : {
381
+ 'landing-page-test.js' : [
382
+ 'should work with passing tests' ,
383
+ // 'should work with skipped tests', // new but not retried because it's skipped
384
+ // 'should work with fixme', // new but not retried because it's skipped
385
+ 'should work with annotated tests'
386
+ ] ,
387
+ 'skipped-suite-test.js' : [
388
+ 'should work with fixme root'
389
+ ] ,
390
+ 'todo-list-page-test.js' : [
391
+ 'should work with failing tests' ,
392
+ 'should work with fixme root'
393
+ ]
394
+ }
395
+ }
396
+ )
397
+
398
+ const receiverPromise = receiver
399
+ . gatherPayloadsMaxTimeout ( ( { url } ) => url === '/api/v2/citestcycle' , ( payloads ) => {
400
+ const events = payloads . flatMap ( ( { payload } ) => payload . events )
401
+ const tests = events . filter ( event => event . type === 'test' ) . map ( event => event . content )
402
+
403
+ const newTests = tests . filter ( test =>
404
+ test . resource ===
405
+ 'landing-page-test.js.should work with skipped tests' ||
406
+ test . resource === 'landing-page-test.js.should work with fixme'
407
+ )
408
+ // no retries
409
+ assert . equal ( newTests . length , 2 )
410
+ newTests . forEach ( test => {
411
+ assert . propertyVal ( test . meta , TEST_IS_NEW , 'true' )
412
+ } )
413
+
414
+ const retriedTests = tests . filter ( test => test . meta [ TEST_IS_RETRY ] === 'true' )
415
+
416
+ assert . equal ( retriedTests . length , 0 )
417
+ } )
418
+
419
+ childProcess = exec (
420
+ './node_modules/.bin/playwright test -c playwright.config.js' ,
421
+ {
422
+ cwd,
423
+ env : {
424
+ ...getCiVisAgentlessConfig ( receiver . port ) ,
425
+ PW_BASE_URL : `http://localhost:${ webAppPort } `
426
+ } ,
427
+ stdio : 'pipe'
428
+ }
429
+ )
430
+
431
+ childProcess . on ( 'exit' , ( ) => {
432
+ receiverPromise . then ( ( ) => done ( ) ) . catch ( done )
433
+ } )
434
+ } )
435
+ } )
436
+ }
220
437
} )
221
438
} )
0 commit comments