diff --git a/src/help.js b/src/help.js index 48679d8..5cb26a5 100644 --- a/src/help.js +++ b/src/help.js @@ -29,6 +29,12 @@ module.exports = { "visible": true, "description": "Retry tests N times (default: 3)." }, + "repetitions": { + "category": "Parallelism, Workflow and Filtering", + "example": "20", + "visible": true, + "description": "Number of times to repeat each test (default: 1)." + }, "test_timeout": { "category": "Parallelism, Workflow and Filtering", "example": "80000", @@ -45,7 +51,7 @@ module.exports = { "category": "Strategy", "example": "testarmada-magellan-locks-resource-strategy", "visible": true, - "description": "The strategy helps magellan hold/release resourcs for test when limit resources are available." + "description": "The strategy helps magellan hold/release resources for test when limit resources are available." }, "debug": { "category": "Parallelism, Workflow and Filtering", diff --git a/src/settings.js b/src/settings.js index cdb4eea..c0a3ac9 100644 --- a/src/settings.js +++ b/src/settings.js @@ -89,6 +89,8 @@ module.exports = { MAX_WORKERS: argv.serial ? 1 : parseInt(argv.max_workers) || 3, MAX_TEST_ATTEMPTS: parseInt(argv.max_test_attempts) || 3, + REPETITIONS: parseInt(argv.repetitions) || 1, + MAX_ALLOCATION_ATTEMPTS: 120, WORKER_START_DELAY: 1000, diff --git a/src/test.js b/src/test.js index 6388b3d..1f04920 100644 --- a/src/test.js +++ b/src/test.js @@ -6,7 +6,7 @@ const TEST_STATUS_SUCCESSFUL = 3; const TEST_STATUS_SKIPPED = 4; class Test { - constructor(locator, profile, executor, maxAttempts) { + constructor(locator, profile, executor, maxAttempts, repetition) { // // note: this locator object is an instance of an object which is defined by whichever test // framework plugin is currently loaded. The implementation of locator could be almost any @@ -23,6 +23,7 @@ class Test { this.profile = profile; this.executor = executor; + this.repetition = repetition; this.workerIndex = -1; this.error = undefined; diff --git a/src/test_runner.js b/src/test_runner.js index c9c09b6..c90dd85 100644 --- a/src/test_runner.js +++ b/src/test_runner.js @@ -53,6 +53,7 @@ class TestRunner { this.strategies = options.strategies; this.MAX_WORKERS = this.settings.MAX_WORKERS; + this.REPETITIONS = this.settings.REPETITIONS; this.MAX_TEST_ATTEMPTS = this.settings.MAX_TEST_ATTEMPTS; @@ -70,14 +71,19 @@ class TestRunner { this.allocator = options.allocator; // For each actual test path, split out - const testsXprofiles = _.flatten( + let testsXprofiles = []; + for (let cnt = 0; cnt < this.REPETITIONS; cnt++) { + testsXprofiles = _.concat(testsXprofiles, _.flatten( tests.map((testLocator) => options.profiles.map((profile) => new Test( testLocator, profile, this.executors[profile.executor], - this.MAX_TEST_ATTEMPTS)))); + this.MAX_TEST_ATTEMPTS, + cnt) + )))); + } if (this.settings.gatherTrends) { this.trends = { @@ -89,6 +95,7 @@ class TestRunner { this.queue = new TestQueue({ tests: testsXprofiles, workerAmount: this.MAX_WORKERS, + repetitions: this.REPETITIONS, stageTestHandler: this.stageTestHandler.bind(this), completeTestHandler: this.completeTestHandler.bind(this), @@ -211,7 +218,8 @@ class TestRunner { const profileStatement = this.profiles.map((b) => b.toString()).join(", "); const serialStatement = this.serial ? "in serial mode" : `with ${this.MAX_WORKERS} workers`; - logger.log(`Running ${this.queue.getTestAmount()} tests` + logger.log(`Running ${this.queue.getTestAmount() / this.REPETITIONS} tests` + + (this.REPETITIONS > 1 ? ` repeated ${this.REPETITIONS} times` : "") + ` ${serialStatement} with [${profileStatement}]`); return this.queue.proceed(); @@ -492,7 +500,10 @@ class TestRunner { // executor: this.executors[test.profile.executor], // The locator object originally generated by the plugin itself - locator: test.locator + locator: test.locator, + + // Which repetition this is + repetition: test.repetition }, ports)); this.setTimeout( diff --git a/test/test_runner.test.js b/test/test_runner.test.js index 481fd73..7caab05 100644 --- a/test/test_runner.test.js +++ b/test/test_runner.test.js @@ -343,6 +343,21 @@ describe("test_runner", () => { jest.runAllTimers(); }); + test("tests are repeated", (done) => { + const testRunner = new TestRunner([stubPassTest, stubPassTest], options, { + settings: { + REPETITIONS: 3 + }, + startTime: (new Date()).getTime() + }); + + testRunner.queue.proceed = jest.fn(); + testRunner.run(); + expect(testRunner.queue.proceed).toHaveBeenCalled(); + expect(testRunner.queue.tests.length).toEqual(6); + done(); + }); + test("test in serial is run successfully", (done) => { fs.mkdirSync.mockImplementation((p) => { }); @@ -666,7 +681,7 @@ describe("test_runner", () => { }); }); -function initTestRunner(tests, options, numPassedTests, numFailedTests) { +function initTestRunner(tests, options) { const testRunner = new TestRunner(tests, options, { settings: { gatherTrends: true, @@ -750,4 +765,4 @@ function stubFailTest() { pass: jest.fn(), fail: jest.fn() } -} \ No newline at end of file +}