Skip to content

Commit 6668493

Browse files
mcasimirmcasimir
authored
chore(ci): run e2e tests in groups COMPASS-7853 (#5924)
* run e2e tests in groups of 3 * add missing variables * also parse test groups if empty string * try remove docker from e2e tests * Update packages/compass-e2e-tests/index.ts * fix test-web-sandbox
1 parent a864151 commit 6668493

File tree

4 files changed

+1222
-207
lines changed

4 files changed

+1222
-207
lines changed

‎.evergreen/buildvariants-and-tasks.in.yml

Lines changed: 127 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ const PACKAGE_BUILD_VARIANTS = [
2727
{
2828
name: 'package-ubuntu',
2929
display_name: 'Package Ubuntu',
30+
31+
// # NOTE: We are packaging on Ubuntu 16.04 in order to use glibc 2.23
32+
// # when compiling/re-building addons. This ensures compatibility with other
33+
// # debian platforms that have glibc 2.23 or newer.
3034
run_on: 'ubuntu1604-large',
3135
},
3236
{
@@ -53,31 +57,31 @@ const PACKAGE_BUILD_VARIANTS = [
5357
5458
const TEST_PACKAGED_APP_BUILD_VARIANTS = [
5559
{
56-
name: 'test-packaged-app-ubuntu',
60+
name: 'test-server-ubuntu',
5761
display_name: 'Ubuntu 20.04',
5862
run_on: 'ubuntu2004-large',
5963
depends_on: 'package-ubuntu',
6064
},
6165
{
62-
name: 'test-packaged-app-windows',
66+
name: 'test-server-windows',
6367
display_name: 'Windows 10',
6468
run_on: 'windows-vsCurrent-large',
6569
depends_on: 'package-windows',
6670
},
6771
{
68-
name: 'test-packaged-app-rhel',
72+
name: 'test-server-rhel',
6973
display_name: 'RHEL 8.0',
7074
run_on: 'rhel80-large',
7175
depends_on: 'package-rhel',
7276
},
7377
{
74-
name: 'test-packaged-app-macos-11-arm',
78+
name: 'test-server-macos-11-arm',
7579
display_name: 'MacOS arm64 11',
7680
run_on: 'macos-1100-arm64-gui',
7781
depends_on: 'package-macos-arm'
7882
},
7983
{
80-
name: 'test-packaged-app-macos-11-x64',
84+
name: 'test-server-macos-11-x64',
8185
display_name: 'MacOS x64 11',
8286
run_on: 'macos-1100-gui',
8387
patchable: false,
@@ -96,8 +100,6 @@ const EOL_SERVER_VERSIONS = [
96100
{ name: '5x-enterprise', version: '5.x.x-enterprise' }
97101
];
98102
99-
// NOTE: the order matters here. The latest element of this array is used
100-
// to test the packaged app.
101103
const MAINTAINED_SERVER_VERSIONS = [
102104
{ name: '60x-community', version: '6.0.x' },
103105
{ name: '60x-enterprise', version: '6.0.x-enterprise' },
@@ -116,44 +118,17 @@ const SERVER_VERSIONS = [
116118
];
117119
118120
const COMPASS_DISTRIBUTIONS = ["compass", "compass-isolated", "compass-readonly"];
121+
122+
const BROWSERS = ["chrome", "firefox"];
123+
124+
const E2E_TEST_GROUPS_NUMBER = 3;
125+
const E2E_TEST_GROUPS = Array.from({ length: E2E_TEST_GROUPS_NUMBER }).map((_, i) => ({number: i + 1}));
119126
%>
120127

121128
#
122129
# Build Variants
123130
#
124131
buildvariants:
125-
- name: connectivity-tests
126-
127-
display_name: Connectivity Tests
128-
run_on: ubuntu2004-large
129-
tasks:
130-
- name: test-connectivity
131-
132-
- name: generate-vulnerability-report
133-
display_name: Vulnerability Report
134-
run_on: ubuntu2004-large
135-
tasks:
136-
- name: generate-vulnerability-report
137-
138-
- name: coverage
139-
display_name: E2E Coverage
140-
run_on: ubuntu2004-large
141-
tasks:
142-
- name: e2e-coverage
143-
144-
- name: csfle-tests
145-
display_name: CSFLE Tests
146-
run_on: ubuntu2004-large
147-
tasks:
148-
- name: test-csfle
149-
150-
- name: test-web-sandbox
151-
display_name: Test Web Sandbox
152-
run_on: ubuntu2004-large
153-
tasks:
154-
- name: test-web-sandbox-chrome
155-
- name: test-web-sandbox-firefox
156-
157132
- name: check
158133
display_name: Check
159134
run_on: ubuntu2004-large
@@ -194,9 +169,11 @@ buildvariants:
194169
- name: package-compass
195170
variant: package-ubuntu
196171
tasks:
197-
<% for (const serverVersion of EOL_SERVER_VERSIONS) { %>
198-
- name: test-packaged-app-<%= serverVersion.name %>
199-
<% } %>
172+
<% for (const serverVersion of EOL_SERVER_VERSIONS) { %>
173+
<% for (const group of E2E_TEST_GROUPS) { %>
174+
- name: test-server-<%= serverVersion.name %>-<%= group.number %>
175+
<% } %>
176+
<% } %>
200177

201178
- name: test-maintained-servers
202179
display_name: Test Maintained Servers
@@ -206,9 +183,11 @@ buildvariants:
206183
- name: package-compass
207184
variant: package-ubuntu
208185
tasks:
209-
<% for (const serverVersion of MAINTAINED_SERVER_VERSIONS) { %>
210-
- name: test-packaged-app-<%= serverVersion.name %>
211-
<% } %>
186+
<% for (const serverVersion of MAINTAINED_SERVER_VERSIONS) { %>
187+
<% for (const group of E2E_TEST_GROUPS) { %>
188+
- name: test-server-<%= serverVersion.name %>-<%= group.number %>
189+
<% } %>
190+
<% } %>
212191

213192
- name: test-latest-alpha
214193
display_name: Test Latest Alpha Server
@@ -218,7 +197,10 @@ buildvariants:
218197
- name: package-compass
219198
variant: package-ubuntu
220199
tasks:
221-
- name: test-packaged-app-<%= TEST_LATEST_ALPHA_SERVER_VERSION.name %>
200+
<% for (const group of E2E_TEST_GROUPS) { %>
201+
- name: test-server-<%= TEST_LATEST_ALPHA_SERVER_VERSION.name %>-<%= group.number %>
202+
<% } %>
203+
222204

223205
# test-packaged-app
224206
<% for (const buildVariant of TEST_PACKAGED_APP_BUILD_VARIANTS) { %>
@@ -230,7 +212,9 @@ buildvariants:
230212
- name: package-compass
231213
variant: <%= buildVariant.depends_on %>
232214
tasks:
233-
- name: test-packaged-app-<%= MAINTAINED_SERVER_VERSIONS[MAINTAINED_SERVER_VERSIONS.length - 1].name %>
215+
<% for (const group of E2E_TEST_GROUPS) { %>
216+
- name: test-packaged-app-<%= group.number %>
217+
<% } %>
234218
<% } %>
235219

236220
- name: publish
@@ -242,6 +226,42 @@ buildvariants:
242226
- name: publish-dev-release-info
243227
- name: create_static_analysis_report
244228

229+
- name: connectivity-tests
230+
display_name: Connectivity Tests
231+
run_on: ubuntu2004-large
232+
tasks:
233+
- name: test-connectivity
234+
235+
- name: e2e-coverage
236+
display_name: E2E Coverage
237+
run_on: ubuntu2004-large
238+
tasks:
239+
<% for(const group of E2E_TEST_GROUPS) { %>
240+
- name: e2e-coverage-<%= group.number %>
241+
<% } %>
242+
243+
- name: csfle-tests
244+
display_name: CSFLE Tests
245+
run_on: ubuntu2004-large
246+
tasks:
247+
- name: test-csfle
248+
249+
- name: test-web-sandbox
250+
display_name: Test Web Sandbox
251+
run_on: ubuntu2004-large
252+
tasks:
253+
<% for (const browser of BROWSERS) { %>
254+
<% for(const group of E2E_TEST_GROUPS) { %>
255+
- name: test-web-sandbox-<%= browser %>-<%= group.number %>
256+
<% } %>
257+
<% } %>
258+
259+
- name: generate-vulnerability-report
260+
display_name: Vulnerability Report
261+
run_on: ubuntu2004-large
262+
tasks:
263+
- name: generate-vulnerability-report
264+
245265
#
246266
# Tasks
247267
#
@@ -292,15 +312,19 @@ tasks:
292312
vars:
293313
debug: 'compass*,electron*,hadron*,mongo*'
294314

295-
- name: e2e-coverage
296-
tags: ['run-on-pr']
315+
<% for(const group of E2E_TEST_GROUPS) { %>
316+
- name: e2e-coverage-<%= group.number %>
317+
tags: ['required-for-publish', 'run-on-pr']
297318
commands:
298319
- func: prepare
299320
- func: install
300321
- func: bootstrap
301322
- func: e2e-coverage
302323
vars:
324+
e2e_test_groups: <%= E2E_TEST_GROUPS.length %>
325+
e2e_test_group: <%= group.number %>
303326
debug: 'compass-e2e-tests*,electron*,hadron*,mongo*'
327+
<% } %>
304328

305329
- name: generate-vulnerability-report
306330
tags: ['required-for-publish', 'run-on-pr']
@@ -350,26 +374,35 @@ tasks:
350374
- func: prepare
351375
- func: publish-dev-release-info
352376

353-
- name: test-web-sandbox-chrome
354-
tags: ['required-for-publish', 'run-on-pr']
377+
<% for (const distribution of COMPASS_DISTRIBUTIONS) { %>
378+
- name: package-<%= distribution %>
379+
tags: ['required-for-publish', 'run-on-pr', 'package-task']
355380
commands:
356381
- func: prepare
357382
- func: install
358383
- func: bootstrap
359384
vars:
360-
scope: 'compass-e2e-tests'
385+
scope: '@mongodb-js/webpack-config-compass'
361386
- func: apply-compass-target-expansion
362387
vars:
363-
compass_distribution: compass
364-
- func: test-web-sandbox
388+
compass_distribution: <%= distribution %>
389+
- func: spawn-signing-server
390+
- func: package
365391
vars:
366-
mongodb_version: 'latest-alpha-enterprise'
367-
browser_name: 'chrome'
368-
compass_distribution: compass
369-
debug: 'compass-e2e-tests*,electron*,hadron*,mongo*'
392+
debug: 'hadron*,mongo*,compass*,electron*,sign*'
393+
compass_distribution: <%= distribution %>
394+
- func: verify-artifacts
395+
- func: save-all-artifacts
396+
vars:
397+
compass_distribution: <%= distribution %>
398+
<% } %>
370399

371-
- name: test-web-sandbox-firefox
372-
tags: ['required-for-publish', 'run-on-pr']
400+
<% for (const serverVersion of SERVER_VERSIONS) { %>
401+
<% for(const group of E2E_TEST_GROUPS) { %>
402+
- name: test-server-<%= serverVersion.name %>-<%= group.number %>
403+
tags:
404+
- required-for-publish
405+
- run-on-pr
373406
commands:
374407
- func: prepare
375408
- func: install
@@ -379,42 +412,49 @@ tasks:
379412
- func: apply-compass-target-expansion
380413
vars:
381414
compass_distribution: compass
382-
- func: test-web-sandbox
415+
- func: get-packaged-app
383416
vars:
384-
mongodb_version: 'latest-alpha-enterprise'
385-
browser_name: 'firefox'
386417
compass_distribution: compass
418+
- func: test-packaged-app
419+
vars:
420+
mongodb_version: '<%= serverVersion.version %>'
421+
compass_distribution: compass
422+
e2e_test_groups: <%= E2E_TEST_GROUPS.length %>
423+
e2e_test_group: <%= group.number %>
387424
debug: 'compass-e2e-tests*,electron*,hadron*,mongo*'
425+
<% } %>
426+
<% } %>
388427

389-
<% for (const distribution of COMPASS_DISTRIBUTIONS) { %>
390-
- name: package-<%= distribution %>
391-
tags: ['required-for-publish', 'run-on-pr', 'package-task']
428+
<% for(const group of E2E_TEST_GROUPS) { %>
429+
- name: test-packaged-app-<%= group.number %>
430+
tags:
431+
- required-for-publish
432+
- run-on-pr
392433
commands:
393434
- func: prepare
394435
- func: install
395436
- func: bootstrap
396437
vars:
397-
scope: '@mongodb-js/webpack-config-compass'
438+
scope: 'compass-e2e-tests'
398439
- func: apply-compass-target-expansion
399440
vars:
400-
compass_distribution: <%= distribution %>
401-
- func: spawn-signing-server
402-
- func: package
441+
compass_distribution: compass
442+
- func: get-packaged-app
403443
vars:
404-
debug: 'hadron*,mongo*,compass*,electron*,sign*'
405-
compass_distribution: <%= distribution %>
406-
- func: verify-artifacts
407-
- func: save-all-artifacts
444+
compass_distribution: compass
445+
- func: test-packaged-app
408446
vars:
409-
compass_distribution: <%= distribution %>
447+
mongodb_version: latest
448+
compass_distribution: compass
449+
e2e_test_groups: <%= E2E_TEST_GROUPS.length %>
450+
e2e_test_group: <%= group.number %>
451+
debug: 'compass-e2e-tests*,electron*,hadron*,mongo*'
410452
<% } %>
411453

412-
<%
413-
for (const serverVersion of SERVER_VERSIONS) { %>
414-
- name: test-packaged-app-<%= serverVersion.name %>
415-
tags:
416-
- required-for-publish
417-
- run-on-pr
454+
<% for (const browser of BROWSERS) { %>
455+
<% for(const group of E2E_TEST_GROUPS) { %>
456+
- name: test-web-sandbox-<%= browser %>-<%= group.number %>
457+
tags: ['required-for-publish', 'run-on-pr']
418458
commands:
419459
- func: prepare
420460
- func: install
@@ -424,14 +464,15 @@ for (const serverVersion of SERVER_VERSIONS) { %>
424464
- func: apply-compass-target-expansion
425465
vars:
426466
compass_distribution: compass
427-
- func: get-packaged-app
428-
vars:
429-
compass_distribution: compass
430-
- func: test-packaged-app
467+
- func: test-web-sandbox
431468
vars:
432-
mongodb_version: '<%= serverVersion.version %>'
469+
mongodb_version: latest
470+
browser_name: '<%= browser %>'
433471
compass_distribution: compass
472+
e2e_test_groups: <%= E2E_TEST_GROUPS.length %>
473+
e2e_test_group: <%= group.number %>
434474
debug: 'compass-e2e-tests*,electron*,hadron*,mongo*'
475+
<% } %>
435476
<% } %>
436477

437478
- name: create_static_analysis_report

0 commit comments

Comments
 (0)