Skip to content
This repository was archived by the owner on Jul 29, 2024. It is now read-only.

Commit 0f7a38a

Browse files
cnishinacnishina
committed
chore(test): error tests fixed (#5069)
- Fix a missing await on createNextTaskRunner where the recursive call should be awaited and if there are no new tasks to still resolve the promise. Things were passing previously probably because we were running tasks out of sync. - Add in errorTest portion of the test suite. - Turn on the angular2 and unit tests. - Turn on browserstack test and remove ciNg2Conf test from Travis.
1 parent 055be2e commit 0f7a38a

25 files changed

+124
-120
lines changed

‎.travis.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,17 @@ env:
1616
matrix:
1717
- JOB=full
1818
- JOB=smoke
19-
# - JOB=bstack
19+
- JOB=bstack
2020

2121
matrix:
2222
allow_failures:
2323
- env: "JOB=smoke"
24-
# - env: "JOB=bstack"
24+
- env: "JOB=bstack"
2525
exclude:
2626
- env: JOB=smoke
2727
node_js: "9"
28-
# - env: JOB=bstack
29-
# node_js: "8"
28+
- env: JOB=bstack
29+
node_js: "9"
3030

3131
addons:
3232
apt:

‎lib/launcher.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ let initFn = async function(configFile: string, additionalConfig: Config) {
221221
}
222222
taskResults_.add(result);
223223
task.done();
224-
createNextTaskRunner();
224+
await createNextTaskRunner();
225225
// If all tasks are finished
226226
if (scheduler.numTasksOutstanding() === 0) {
227227
resolve();
@@ -232,6 +232,8 @@ let initFn = async function(configFile: string, additionalConfig: Config) {
232232
logger.error('Error:', (err as any).stack || err.message || err);
233233
await cleanUpAndExit(errorCode ? errorCode : RUNNERS_FAILED_EXIT_CODE);
234234
}
235+
} else {
236+
resolve();
235237
}
236238
});
237239
};

‎scripts/test.js

+44-44
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,15 @@ const passingTests = [
3838
'node built/cli.js spec/controlLockConf.js',
3939
'node built/cli.js spec/customFramework.js',
4040
'node built/cli.js spec/noGlobalsConf.js',
41-
// 'node built/cli.js spec/angular2Conf.js',
41+
'node built/cli.js spec/angular2Conf.js',
4242
'node built/cli.js spec/hybridConf.js',
4343
'node built/cli.js spec/built/noCFBasicConf.js',
4444
'node built/cli.js spec/built/noCFBasicConf.js --useBlockingProxy',
4545
'node built/cli.js spec/built/noCFPluginConf.js',
4646
'node scripts/driverProviderAttachSession.js',
47-
// 'node scripts/errorTest.js',
48-
// // Unit tests
49-
// 'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/unit_test.json',
47+
'node scripts/errorTest.js',
48+
// Unit tests
49+
'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/unit_test.json',
5050
// Dependency tests
5151
'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/dependency_test.json',
5252
// Typings tests
@@ -68,7 +68,7 @@ passingTests.forEach((passing_test) => {
6868
executor.addCommandlineTest('node built/cli.js spec/errorTest/singleFailureConf.js')
6969
.expectExitCode(1)
7070
.expectErrors({
71-
stackTrace: 'single_failure_spec1.js:5:32'
71+
stackTrace: 'single_failure_spec1.js:5:38'
7272
});
7373

7474
// assert timeout works
@@ -90,63 +90,63 @@ executor.addCommandlineTest('node built/cli.js spec/errorTest/multiFailureConf.j
9090
.expectExitCode(1)
9191
.expectErrors([{
9292
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
93-
stacktrace: 'single_failure_spec1.js:5:32'
93+
stacktrace: 'single_failure_spec1.js:5:38'
9494
}, {
9595
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
96-
stacktrace: 'single_failure_spec2.js:5:32'
96+
stacktrace: 'single_failure_spec2.js:5:38'
9797
}]);
9898

9999
executor.addCommandlineTest('node built/cli.js spec/errorTest/shardedFailureConf.js')
100100
.expectExitCode(1)
101101
.expectErrors([{
102102
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
103-
stacktrace: 'single_failure_spec1.js:5:32'
103+
stacktrace: 'single_failure_spec1.js:5:38'
104104
}, {
105105
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
106-
stacktrace: 'single_failure_spec2.js:5:32'
106+
stacktrace: 'single_failure_spec2.js:5:38'
107107
}]);
108108

109109
executor.addCommandlineTest('node built/cli.js spec/errorTest/mochaFailureConf.js')
110110
.expectExitCode(1)
111111
.expectErrors([{
112112
message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'',
113-
stacktrace: 'mocha_failure_spec.js:11:20'
113+
stacktrace: 'mocha_failure_spec.js:11:41'
114114
}]);
115115

116-
// executor.addCommandlineTest('node built/cli.js spec/errorTest/pluginsFailingConf.js')
117-
// .expectExitCode(1)
118-
// .expectErrors([
119-
// {message: 'Expected true to be false'},
120-
// {message: 'from setup'},
121-
// {message: 'from postTest passing'},
122-
// {message: 'from postTest failing'},
123-
// {message: 'from teardown'}
124-
// ]);
125-
126-
// executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js')
127-
// .expectExitCode(1)
128-
// .expectErrors([
129-
// {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
130-
// {message: 'The following tasks were pending:[\\s\\S]*' +
131-
// '- \\$timeout: function\\(\\) {[\\s\\S]*' +
132-
// '\\$scope\\.slowAngularTimeoutStatus = \'done\';[\\s\\S]' +
133-
// '*}'}
134-
// ]);
135-
136-
// executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js ' +
137-
// '--untrackOutstandingTimeouts true')
138-
// .expectExitCode(1)
139-
// .expectErrors([
140-
// {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
141-
// {message: 'While waiting for element with locator - ' +
142-
// 'Locator: by.binding\\(\\"slowAngularTimeoutStatus\\"\\)$'}
143-
// ]);
144-
145-
// executor.addCommandlineTest('node built/cli.js spec/angular2TimeoutConf.js')
146-
// .expectExitCode(1)
147-
// .expectErrors([
148-
// {message: 'Timed out waiting for asynchronous Angular tasks to finish'},
149-
// ]);
116+
executor.addCommandlineTest('node built/cli.js spec/errorTest/pluginsFailingConf.js')
117+
.expectExitCode(1)
118+
.expectErrors([
119+
{message: 'Expected true to be false'},
120+
{message: 'from setup'},
121+
{message: 'from postTest passing'},
122+
{message: 'from postTest failing'},
123+
{message: 'from teardown'}
124+
]);
125+
126+
executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js')
127+
.expectExitCode(1)
128+
.expectErrors([
129+
{message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
130+
{message: 'The following tasks were pending:[\\s\\S]*' +
131+
'- \\$timeout: function\\(\\) {[\\s\\S]*' +
132+
'\\$scope\\.slowAngularTimeoutStatus = \'done\';[\\s\\S]' +
133+
'*}'}
134+
]);
135+
136+
executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js ' +
137+
'--untrackOutstandingTimeouts true')
138+
.expectExitCode(1)
139+
.expectErrors([
140+
{message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
141+
{message: 'While waiting for element with locator - ' +
142+
'Locator: by.binding\\(\\"slowAngularTimeoutStatus\\"\\)$'}
143+
]);
144+
145+
executor.addCommandlineTest('node built/cli.js spec/angular2TimeoutConf.js')
146+
.expectExitCode(1)
147+
.expectErrors([
148+
{message: 'Timed out waiting for asynchronous Angular tasks to finish'},
149+
]);
150150

151151
// If we're running on CircleCI, save stdout and stderr from the test run to a log file.
152152
if (process.env['CIRCLE_ARTIFACTS']) {

‎scripts/test_on_travis.sh

+5-5
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ if [ $JOB = "smoke" ]; then
55
node bin/protractor spec/ciSmokeConf.js
66
elif [ $JOB = "full" ]; then
77
node bin/protractor spec/ciFullConf.js
8-
if [ $? = "0" ]; then
9-
node bin/protractor spec/ciNg2Conf.js
10-
else
11-
exit 1
12-
fi
8+
# if [ $? = "0" ]; then
9+
# node bin/protractor spec/ciNg2Conf.js
10+
# else
11+
# exit 1
12+
# fi
1313
elif [ $JOB = "bstack" ]; then
1414
node bin/protractor spec/ciBStackConf.js
1515
else

‎spec/ciFullConf.js

+1-11
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,8 @@ exports.config = {
99
framework: 'jasmine',
1010

1111
// Spec patterns are relative to this directory.
12-
// TODO(selenium4): revert back to basic/*_spec.js
1312
specs: [
14-
'basic/lib_spec.js',
15-
'basic/locators_spec.js'
16-
// 'basic/elements_spec.js',
17-
// 'basic/expected_conditions_spec.js',
18-
// 'basic/handling_spec.js'
19-
// 'basic/mockmodule_spec.js',
20-
// 'basic/navigation_spec.js',
21-
// 'basic/polling_spec.js',
22-
// 'basic/restart_spec.js',
23-
// 'basic/synchronize_spec.js',
13+
'basic/*_spec.js',
2414
],
2515

2616
// Exclude patterns are relative to this directory.

‎spec/errorTest/afterLaunchChangesExitCodeConf.js

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ var env = require('../environment.js');
22

33
exports.config = {
44
seleniumAddress: env.seleniumAddress,
5+
SELENIUM_PROMISE_MANAGER: false,
56

67
framework: 'jasmine',
78

‎spec/errorTest/baseCase/error_spec.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
describe('finding an element that does not exist', function() {
2-
it('should throw an error', function() {
3-
browser.get('index.html');
1+
describe('finding an element that does not exist', () => {
2+
it('should throw an error', async () => {
3+
await browser.get('index.html');
44
element(by.binding('INVALID')); // greeting
55
});
66
});

‎spec/errorTest/baseCase/mocha_failure_spec.js

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
// Use the external Chai As Promised to deal with resolving promises in
22
// expectations.
3-
var chai = require('chai');
4-
var chaiAsPromised = require('chai-as-promised');
3+
const chai = require('chai');
4+
const chaiAsPromised = require('chai-as-promised');
55
chai.use(chaiAsPromised);
6-
var expect = chai.expect;
6+
const expect = chai.expect;
77

8-
describe('protractor library', function() {
9-
it('should fail', function() {
10-
browser.get('index.html');
11-
expect(browser.getTitle()).to.eventually.equal('INTENTIONALLY INCORRECT');
8+
describe('protractor library', () => {
9+
it('should fail', async () => {
10+
await browser.get('index.html');
11+
expect(await browser.getTitle()).to.equal('INTENTIONALLY INCORRECT');
1212
});
1313
});

‎spec/errorTest/baseCase/single_failure_spec1.js

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
describe('single failure spec1', function() {
2-
it('should fail expectation', function() {
3-
browser.get('index.html');
4-
var greeting = element(by.binding('greeting'));
5-
expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
1+
describe('single failure spec1', () => {
2+
it('should fail expectation', async () => {
3+
await browser.get('index.html');
4+
const greeting = element(by.binding('greeting'));
5+
expect(await greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
66
});
77
});

‎spec/errorTest/baseCase/single_failure_spec2.js

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
describe('single failure spec2', function() {
2-
it('should fail expectation', function() {
3-
browser.get('index.html');
4-
var greeting = element(by.binding('greeting'));
5-
expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
1+
describe('single failure spec2', () => {
2+
it('should fail expectation', async () => {
3+
await browser.get('index.html');
4+
const greeting = element(by.binding('greeting'));
5+
expect(await greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
66
});
77
});

‎spec/errorTest/baseCase/slow_http_and_timeout_spec.js

+15-15
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,27 @@
1-
describe('slow asynchronous events', function() {
2-
beforeEach(function() {
3-
browser.get('index.html#/async');
1+
describe('slow asynchronous events', () => {
2+
beforeEach(async () => {
3+
await browser.get('index.html#/async');
44
});
55

6-
it('waits for http calls', function() {
7-
var status = element(by.binding('slowHttpStatus'));
8-
var button = element(by.css('[ng-click="slowHttp()"]'));
6+
it('waits for http calls', async () => {
7+
const status = element(by.binding('slowHttpStatus'));
8+
const button = element(by.css('[ng-click="slowHttp()"]'));
99

10-
expect(status.getText()).toEqual('not started');
10+
expect(await status.getText()).toEqual('not started');
1111

12-
button.click();
12+
await button.click();
1313

14-
expect(status.getText()).toEqual('done');
14+
expect(await status.getText()).toEqual('done');
1515
});
1616

17-
it('waits for $timeout', function() {
18-
var status = element(by.binding('slowAngularTimeoutStatus'));
19-
var button = element(by.css('[ng-click="slowAngularTimeout()"]'));
17+
it('waits for $timeout', async () => {
18+
const status = element(by.binding('slowAngularTimeoutStatus'));
19+
const button = element(by.css('[ng-click="slowAngularTimeout()"]'));
2020

21-
expect(status.getText()).toEqual('not started');
21+
expect(await status.getText()).toEqual('not started');
2222

23-
button.click();
23+
await button.click();
2424

25-
expect(status.getText()).toEqual('done');
25+
expect(await status.getText()).toEqual('done');
2626
});
2727
});

‎spec/errorTest/baseCase/success_spec.js

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
describe('success spec', function() {
2-
it('should pass', function() {
3-
browser.get('index.html');
4-
var greeting = element(by.binding('greeting'));
5-
expect(greeting.getText()).toEqual('Hiya');
1+
describe('success spec', () => {
2+
it('should pass', async () => {
3+
await browser.get('index.html');
4+
const greeting = element(by.binding('greeting'));
5+
expect(await greeting.getText()).toEqual('Hiya');
66
});
77
});

‎spec/errorTest/baseCase/timeout_spec.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
describe('timeout spec', function() {
2-
it('should timeout due to jasmine spec limit', function() {
3-
browser.get('index.html#/form');
1+
describe('timeout spec', () => {
2+
it('should timeout due to jasmine spec limit', async () => {
3+
await browser.get('index.html#/form');
44
}, 1);
55
});

‎spec/errorTest/browserStackAuthentication.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
var env = require('../environment.js');
1+
const env = require('../environment.js');
22

33
exports.config = {
44
browserstackUser: 'foobar',
55
browserstackKey: 'foobar',
6+
SELENIUM_PROMISE_MANAGER: false,
67

78
framework: 'jasmine',
89

‎spec/errorTest/debugMultiCapabilities.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
var env = require('../environment.js');
1+
const env = require('../environment.js');
22

33
exports.config = {
44
seleniumAddress: env.seleniumAddress,
5+
SELENIUM_PROMISE_MANAGER: false,
56
framework: 'jasmine',
67
debug: true,
78
specs: [

‎spec/errorTest/getMultiCapabilitiesConf.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
var env = require('../environment.js');
1+
const env = require('../environment.js');
22

33
exports.config = {
44
seleniumAddress: env.seleniumAddress,
5+
SELENIUM_PROMISE_MANAGER: false,
56

67
// Spec patterns are relative to this directory.
78
specs: [

‎spec/errorTest/mochaFailureConf.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
var env = require('../environment.js');
1+
const env = require('../environment.js');
22

33
exports.config = {
44
seleniumAddress: env.seleniumAddress,
5+
SELENIUM_PROMISE_MANAGER: false,
56

67
specs: [
78
'baseCase/mocha_failure_spec.js'

‎spec/errorTest/multiFailureConf.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
var env = require('../environment.js');
1+
const env = require('../environment.js');
22

33
exports.config = {
44
seleniumAddress: env.seleniumAddress,
5+
SELENIUM_PROMISE_MANAGER: false,
56

67
framework: 'jasmine',
78

0 commit comments

Comments
 (0)