Skip to content

Commit 9fd7070

Browse files
dplewisdplewis
authored
ci: Add test retry logic for flaky tests (#9218)
1 parent 453a987 commit 9fd7070

File tree

5 files changed

+129
-42
lines changed

5 files changed

+129
-42
lines changed

‎spec/Idempotency.spec.js

+4
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ describe('Idempotency', () => {
4545
});
4646
});
4747

48+
afterEach(() => {
49+
jasmine.DEFAULT_TIMEOUT_INTERVAL = process.env.PARSE_SERVER_TEST_TIMEOUT || 10000;
50+
});
51+
4852
// Tests
4953
it_id('e25955fd-92eb-4b22-b8b7-38980e5cb223')('should enforce idempotency for cloud code function', async () => {
5054
let counter = 0;

‎spec/RegexVulnerabilities.spec.js

+38-33
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,13 @@ const emailAdapter = {
1616
const appName = 'test';
1717
const publicServerURL = 'http://localhost:8378/1';
1818

19-
describe('Regex Vulnerabilities', function () {
20-
beforeEach(async function () {
19+
describe('Regex Vulnerabilities', () => {
20+
let objectId;
21+
let sessionToken;
22+
let partialSessionToken;
23+
let user;
24+
25+
beforeEach(async () => {
2126
await reconfigureServer({
2227
maintenanceKey: 'test2',
2328
verifyUserEmails: true,
@@ -38,13 +43,13 @@ describe('Regex Vulnerabilities', function () {
3843
email: '[email protected]',
3944
}),
4045
});
41-
this.objectId = signUpResponse.data.objectId;
42-
this.sessionToken = signUpResponse.data.sessionToken;
43-
this.partialSessionToken = this.sessionToken.slice(0, 3);
46+
objectId = signUpResponse.data.objectId;
47+
sessionToken = signUpResponse.data.sessionToken;
48+
partialSessionToken = sessionToken.slice(0, 3);
4449
});
4550

46-
describe('on session token', function () {
47-
it('should not work with regex', async function () {
51+
describe('on session token', () => {
52+
it('should not work with regex', async () => {
4853
try {
4954
await request({
5055
url: `${serverURL}/users/me`,
@@ -53,7 +58,7 @@ describe('Regex Vulnerabilities', function () {
5358
body: JSON.stringify({
5459
...keys,
5560
_SessionToken: {
56-
$regex: this.partialSessionToken,
61+
$regex: partialSessionToken,
5762
},
5863
_method: 'GET',
5964
}),
@@ -65,43 +70,43 @@ describe('Regex Vulnerabilities', function () {
6570
}
6671
});
6772

68-
it('should work with plain token', async function () {
73+
it('should work with plain token', async () => {
6974
const meResponse = await request({
7075
url: `${serverURL}/users/me`,
7176
method: 'POST',
7277
headers,
7378
body: JSON.stringify({
7479
...keys,
75-
_SessionToken: this.sessionToken,
80+
_SessionToken: sessionToken,
7681
_method: 'GET',
7782
}),
7883
});
79-
expect(meResponse.data.objectId).toEqual(this.objectId);
80-
expect(meResponse.data.sessionToken).toEqual(this.sessionToken);
84+
expect(meResponse.data.objectId).toEqual(objectId);
85+
expect(meResponse.data.sessionToken).toEqual(sessionToken);
8186
});
8287
});
8388

84-
describe('on verify e-mail', function () {
89+
describe('on verify e-mail', () => {
8590
beforeEach(async function () {
8691
const userQuery = new Parse.Query(Parse.User);
87-
this.user = await userQuery.get(this.objectId, { useMasterKey: true });
92+
user = await userQuery.get(objectId, { useMasterKey: true });
8893
});
8994

90-
it('should not work with regex', async function () {
91-
expect(this.user.get('emailVerified')).toEqual(false);
95+
it('should not work with regex', async () => {
96+
expect(user.get('emailVerified')).toEqual(false);
9297
await request({
9398
url: `${serverURL}/apps/test/[email protected]&token[$regex]=`,
9499
method: 'GET',
95100
});
96-
await this.user.fetch({ useMasterKey: true });
97-
expect(this.user.get('emailVerified')).toEqual(false);
101+
await user.fetch({ useMasterKey: true });
102+
expect(user.get('emailVerified')).toEqual(false);
98103
});
99104

100-
it_id('92bbb86d-bcda-49fa-8d79-aa0501078044')('should work with plain token', async function () {
101-
expect(this.user.get('emailVerified')).toEqual(false);
105+
it_id('92bbb86d-bcda-49fa-8d79-aa0501078044')('should work with plain token', async () => {
106+
expect(user.get('emailVerified')).toEqual(false);
102107
const current = await request({
103108
method: 'GET',
104-
url: `http://localhost:8378/1/classes/_User/${this.user.id}`,
109+
url: `http://localhost:8378/1/classes/_User/${user.id}`,
105110
json: true,
106111
headers: {
107112
'X-Parse-Application-Id': 'test',
@@ -115,18 +120,18 @@ describe('Regex Vulnerabilities', function () {
115120
url: `${serverURL}/apps/test/[email protected]&token=${current._email_verify_token}`,
116121
method: 'GET',
117122
});
118-
await this.user.fetch({ useMasterKey: true });
119-
expect(this.user.get('emailVerified')).toEqual(true);
123+
await user.fetch({ useMasterKey: true });
124+
expect(user.get('emailVerified')).toEqual(true);
120125
});
121126
});
122127

123-
describe('on password reset', function () {
124-
beforeEach(async function () {
125-
this.user = await Parse.User.logIn('[email protected]', 'somepassword');
128+
describe('on password reset', () => {
129+
beforeEach(async () => {
130+
user = await Parse.User.logIn('[email protected]', 'somepassword');
126131
});
127132

128-
it('should not work with regex', async function () {
129-
expect(this.user.id).toEqual(this.objectId);
133+
it('should not work with regex', async () => {
134+
expect(user.id).toEqual(objectId);
130135
await request({
131136
url: `${serverURL}/requestPasswordReset`,
132137
method: 'POST',
@@ -137,7 +142,7 @@ describe('Regex Vulnerabilities', function () {
137142
email: '[email protected]',
138143
}),
139144
});
140-
await this.user.fetch({ useMasterKey: true });
145+
await user.fetch({ useMasterKey: true });
141146
const passwordResetResponse = await request({
142147
url: `${serverURL}/apps/test/[email protected]&token[$regex]=`,
143148
method: 'GET',
@@ -162,8 +167,8 @@ describe('Regex Vulnerabilities', function () {
162167
}
163168
});
164169

165-
it('should work with plain token', async function () {
166-
expect(this.user.id).toEqual(this.objectId);
170+
it('should work with plain token', async () => {
171+
expect(user.id).toEqual(objectId);
167172
await request({
168173
url: `${serverURL}/requestPasswordReset`,
169174
method: 'POST',
@@ -176,7 +181,7 @@ describe('Regex Vulnerabilities', function () {
176181
});
177182
const current = await request({
178183
method: 'GET',
179-
url: `http://localhost:8378/1/classes/_User/${this.user.id}`,
184+
url: `http://localhost:8378/1/classes/_User/${user.id}`,
180185
json: true,
181186
headers: {
182187
'X-Parse-Application-Id': 'test',
@@ -204,7 +209,7 @@ describe('Regex Vulnerabilities', function () {
204209
},
205210
});
206211
const userAgain = await Parse.User.logIn('[email protected]', 'newpassword');
207-
expect(userAgain.id).toEqual(this.objectId);
212+
expect(userAgain.id).toEqual(objectId);
208213
});
209214
});
210215
});

‎spec/helper.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ if (dns.setDefaultResultOrder) {
1414
jasmine.DEFAULT_TIMEOUT_INTERVAL = process.env.PARSE_SERVER_TEST_TIMEOUT || 10000;
1515
jasmine.getEnv().addReporter(new CurrentSpecReporter());
1616
jasmine.getEnv().addReporter(new SpecReporter());
17+
global.retryFlakyTests();
1718

1819
global.on_db = (db, callback, elseCallback) => {
1920
if (process.env.PARSE_SERVER_TEST_DB == db) {
@@ -287,7 +288,7 @@ afterEach(function (done) {
287288
});
288289

289290
afterAll(() => {
290-
global.displaySlowTests();
291+
global.displayTestStats();
291292
});
292293

293294
const TestObject = Parse.Object.extend({

‎spec/support/CurrentSpecReporter.js

+84-7
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,36 @@
11
// Sets a global variable to the current test spec
22
// ex: global.currentSpec.description
33
const { performance } = require('perf_hooks');
4+
45
global.currentSpec = null;
56

6-
const timerMap = {};
7-
const duplicates = [];
7+
/**
8+
* Names of tests that fail randomly and are considered flaky. These tests will be retried
9+
* a number of times to reduce the chance of false negatives. The test name must be the same
10+
* as the one displayed in the CI log test output.
11+
*/
12+
const flakyTests = [
13+
// Timeout
14+
"ParseLiveQuery handle invalid websocket payload length",
15+
// Unhandled promise rejection: TypeError: message.split is not a function
16+
"rest query query internal field",
17+
// TypeError: Cannot read properties of undefined (reading 'link')
18+
"UserController sendVerificationEmail parseFrameURL not provided uses publicServerURL",
19+
// TypeError: Cannot read properties of undefined (reading 'link')
20+
"UserController sendVerificationEmail parseFrameURL provided uses parseFrameURL and includes the destination in the link parameter",
21+
// Expected undefined to be defined
22+
"Email Verification Token Expiration: sets the _email_verify_token_expires_at and _email_verify_token fields after user SignUp",
23+
];
24+
825
/** The minimum execution time in seconds for a test to be considered slow. */
926
const slowTestLimit = 2;
1027

28+
/** The number of times to retry a flaky test. */
29+
const retries = 5;
30+
31+
const timerMap = {};
32+
const retryMap = {};
33+
const duplicates = [];
1134
class CurrentSpecReporter {
1235
specStarted(spec) {
1336
if (timerMap[spec.fullName]) {
@@ -26,20 +49,74 @@ class CurrentSpecReporter {
2649
global.currentSpec = null;
2750
}
2851
}
29-
global.displaySlowTests = function() {
30-
const times = Object.values(timerMap).sort((a,b) => b - a);
52+
53+
global.displayTestStats = function() {
54+
const times = Object.values(timerMap).sort((a,b) => b - a).filter(time => time >= slowTestLimit);
3155
if (times.length > 0) {
3256
console.log(`Slow tests with execution time >=${slowTestLimit}s:`);
3357
}
3458
times.forEach((time) => {
35-
if (time >= slowTestLimit) {
36-
console.warn(`${time.toFixed(1)}s:`, Object.keys(timerMap).find(key => timerMap[key] === time));
37-
}
59+
console.warn(`${time.toFixed(1)}s:`, Object.keys(timerMap).find(key => timerMap[key] === time));
3860
});
3961
console.log('\n');
4062
duplicates.forEach((spec) => {
4163
console.warn('Duplicate spec: ' + spec);
4264
});
65+
console.log('\n');
66+
Object.keys(retryMap).forEach((spec) => {
67+
console.warn(`Flaky test: ${spec} failed ${retryMap[spec]} times`);
68+
});
69+
console.log('\n');
4370
};
4471

72+
global.retryFlakyTests = function() {
73+
const originalSpecConstructor = jasmine.Spec;
74+
75+
jasmine.Spec = function(attrs) {
76+
const spec = new originalSpecConstructor(attrs);
77+
const originalTestFn = spec.queueableFn.fn;
78+
const runOriginalTest = () => {
79+
if (originalTestFn.length == 0) {
80+
// handle async testing
81+
return originalTestFn();
82+
} else {
83+
// handle done() callback
84+
return new Promise((resolve) => {
85+
originalTestFn(resolve);
86+
});
87+
}
88+
};
89+
spec.queueableFn.fn = async function() {
90+
const isFlaky = flakyTests.includes(spec.result.fullName);
91+
const runs = isFlaky ? retries : 1;
92+
let exceptionCaught;
93+
let returnValue;
94+
95+
for (let i = 0; i < runs; ++i) {
96+
spec.result.failedExpectations = [];
97+
returnValue = undefined;
98+
exceptionCaught = undefined;
99+
try {
100+
returnValue = await runOriginalTest();
101+
} catch (exception) {
102+
exceptionCaught = exception;
103+
}
104+
const failed = !spec.markedPending &&
105+
(exceptionCaught || spec.result.failedExpectations.length != 0);
106+
if (!failed) {
107+
break;
108+
}
109+
if (isFlaky) {
110+
retryMap[spec.result.fullName] = (retryMap[spec.result.fullName] || 0) + 1;
111+
}
112+
}
113+
if (exceptionCaught) {
114+
throw exceptionCaught;
115+
}
116+
return returnValue;
117+
};
118+
return spec;
119+
};
120+
}
121+
45122
module.exports = CurrentSpecReporter;

‎spec/support/jasmine.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"spec_dir": "spec",
33
"spec_files": ["*spec.js"],
44
"helpers": ["helper.js"],
5-
"random": false
5+
"random": true
66
}

0 commit comments

Comments
 (0)