Skip to content

Commit 2d874ad

Browse files
juan-fernandezjuan-fernandez
authored
[ci-visibility] Early flake detection for playwright (#4143)
1 parent 6227fa4 commit 2d874ad

File tree

3 files changed

+379
-9
lines changed

3 files changed

+379
-9
lines changed

‎integration-tests/playwright/playwright.spec.js

Lines changed: 218 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@ const {
1717
TEST_SOURCE_START,
1818
TEST_TYPE,
1919
TEST_SOURCE_FILE,
20-
TEST_CONFIGURATION_BROWSER_NAME
20+
TEST_CONFIGURATION_BROWSER_NAME,
21+
TEST_IS_NEW,
22+
TEST_IS_RETRY
2123
} = require('../../packages/dd-trace/src/plugins/util/test')
2224
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
2325

26+
const NUM_RETRIES_EFD = 3
27+
2428
const versions = ['1.18.0', 'latest']
2529

2630
versions.forEach((version) => {
@@ -217,5 +221,218 @@ versions.forEach((version) => {
217221
}
218222
)
219223
})
224+
225+
if (version === 'latest') {
226+
context('early flake detection', () => {
227+
it('retries new tests', (done) => {
228+
receiver.setSettings({
229+
itr_enabled: false,
230+
code_coverage: false,
231+
tests_skipping: false,
232+
early_flake_detection: {
233+
enabled: true,
234+
slow_test_retries: {
235+
'5s': NUM_RETRIES_EFD
236+
}
237+
}
238+
})
239+
240+
receiver.setKnownTests(
241+
{
242+
playwright: {
243+
'landing-page-test.js': [
244+
// 'should work with passing tests', // it will be considered new
245+
'should work with skipped tests',
246+
'should work with fixme',
247+
'should work with annotated tests'
248+
],
249+
'skipped-suite-test.js': [
250+
'should work with fixme root'
251+
],
252+
'todo-list-page-test.js': [
253+
'should work with failing tests',
254+
'should work with fixme root'
255+
]
256+
}
257+
}
258+
)
259+
260+
const receiverPromise = receiver
261+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
262+
const events = payloads.flatMap(({ payload }) => payload.events)
263+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
264+
265+
const newTests = tests.filter(test =>
266+
test.resource ===
267+
'landing-page-test.js.should work with passing tests'
268+
)
269+
newTests.forEach(test => {
270+
assert.propertyVal(test.meta, TEST_IS_NEW, 'true')
271+
})
272+
273+
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
274+
275+
assert.equal(retriedTests.length, NUM_RETRIES_EFD)
276+
277+
// all but one has been retried
278+
assert.equal(retriedTests.length, newTests.length - 1)
279+
})
280+
281+
childProcess = exec(
282+
'./node_modules/.bin/playwright test -c playwright.config.js',
283+
{
284+
cwd,
285+
env: {
286+
...getCiVisAgentlessConfig(receiver.port),
287+
PW_BASE_URL: `http://localhost:${webAppPort}`
288+
},
289+
stdio: 'pipe'
290+
}
291+
)
292+
293+
childProcess.on('exit', () => {
294+
receiverPromise.then(() => done()).catch(done)
295+
})
296+
})
297+
it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => {
298+
receiver.setSettings({
299+
itr_enabled: false,
300+
code_coverage: false,
301+
tests_skipping: false,
302+
early_flake_detection: {
303+
enabled: true,
304+
slow_test_retries: {
305+
'5s': NUM_RETRIES_EFD
306+
}
307+
}
308+
})
309+
310+
receiver.setKnownTests(
311+
{
312+
playwright: {
313+
'landing-page-test.js': [
314+
// 'should work with passing tests', // it will be considered new
315+
'should work with skipped tests',
316+
'should work with fixme',
317+
'should work with annotated tests'
318+
],
319+
'skipped-suite-test.js': [
320+
'should work with fixme root'
321+
],
322+
'todo-list-page-test.js': [
323+
'should work with failing tests',
324+
'should work with fixme root'
325+
]
326+
}
327+
}
328+
)
329+
330+
const receiverPromise = receiver
331+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
332+
const events = payloads.flatMap(({ payload }) => payload.events)
333+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
334+
335+
const newTests = tests.filter(test =>
336+
test.resource ===
337+
'landing-page-test.js.should work with passing tests'
338+
)
339+
newTests.forEach(test => {
340+
assert.notProperty(test.meta, TEST_IS_NEW)
341+
})
342+
343+
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
344+
345+
assert.equal(retriedTests.length, 0)
346+
})
347+
348+
childProcess = exec(
349+
'./node_modules/.bin/playwright test -c playwright.config.js',
350+
{
351+
cwd,
352+
env: {
353+
...getCiVisAgentlessConfig(receiver.port),
354+
PW_BASE_URL: `http://localhost:${webAppPort}`,
355+
DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false'
356+
},
357+
stdio: 'pipe'
358+
}
359+
)
360+
361+
childProcess.on('exit', () => {
362+
receiverPromise.then(() => done()).catch(done)
363+
})
364+
})
365+
it('does not retry tests that are skipped', (done) => {
366+
receiver.setSettings({
367+
itr_enabled: false,
368+
code_coverage: false,
369+
tests_skipping: false,
370+
early_flake_detection: {
371+
enabled: true,
372+
slow_test_retries: {
373+
'5s': NUM_RETRIES_EFD
374+
}
375+
}
376+
})
377+
378+
receiver.setKnownTests(
379+
{
380+
playwright: {
381+
'landing-page-test.js': [
382+
'should work with passing tests',
383+
// 'should work with skipped tests', // new but not retried because it's skipped
384+
// 'should work with fixme', // new but not retried because it's skipped
385+
'should work with annotated tests'
386+
],
387+
'skipped-suite-test.js': [
388+
'should work with fixme root'
389+
],
390+
'todo-list-page-test.js': [
391+
'should work with failing tests',
392+
'should work with fixme root'
393+
]
394+
}
395+
}
396+
)
397+
398+
const receiverPromise = receiver
399+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
400+
const events = payloads.flatMap(({ payload }) => payload.events)
401+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
402+
403+
const newTests = tests.filter(test =>
404+
test.resource ===
405+
'landing-page-test.js.should work with skipped tests' ||
406+
test.resource === 'landing-page-test.js.should work with fixme'
407+
)
408+
// no retries
409+
assert.equal(newTests.length, 2)
410+
newTests.forEach(test => {
411+
assert.propertyVal(test.meta, TEST_IS_NEW, 'true')
412+
})
413+
414+
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
415+
416+
assert.equal(retriedTests.length, 0)
417+
})
418+
419+
childProcess = exec(
420+
'./node_modules/.bin/playwright test -c playwright.config.js',
421+
{
422+
cwd,
423+
env: {
424+
...getCiVisAgentlessConfig(receiver.port),
425+
PW_BASE_URL: `http://localhost:${webAppPort}`
426+
},
427+
stdio: 'pipe'
428+
}
429+
)
430+
431+
childProcess.on('exit', () => {
432+
receiverPromise.then(() => done()).catch(done)
433+
})
434+
})
435+
})
436+
}
220437
})
221438
})

0 commit comments

Comments
 (0)