From a6c9a34465dbd19ab83a5d62b50dde7dd3aea0bd Mon Sep 17 00:00:00 2001 From: brimstedt Date: Mon, 24 Sep 2018 22:34:35 +0200 Subject: [PATCH 1/5] Added repetitions --- src/help.js | 6 ++++++ src/settings.js | 2 ++ src/test_runner.js | 15 ++++++++++----- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/help.js b/src/help.js index 48679d8..0c01eb8 100644 --- a/src/help.js +++ b/src/help.js @@ -29,6 +29,12 @@ module.exports = { "visible": true, "description": "Retry tests N times (default: 3)." }, + "repetitions": { + "category": "Parallelism, Workflow and Filtering", + "example": "20", + "visible": true, + "description": "Number of times to repeat each test (default: 1)." + }, "test_timeout": { "category": "Parallelism, Workflow and Filtering", "example": "80000", diff --git a/src/settings.js b/src/settings.js index cdb4eea..c0a3ac9 100644 --- a/src/settings.js +++ b/src/settings.js @@ -89,6 +89,8 @@ module.exports = { MAX_WORKERS: argv.serial ? 1 : parseInt(argv.max_workers) || 3, MAX_TEST_ATTEMPTS: parseInt(argv.max_test_attempts) || 3, + REPETITIONS: parseInt(argv.repetitions) || 1, + MAX_ALLOCATION_ATTEMPTS: 120, WORKER_START_DELAY: 1000, diff --git a/src/test_runner.js b/src/test_runner.js index c9c09b6..943a0e5 100644 --- a/src/test_runner.js +++ b/src/test_runner.js @@ -53,6 +53,7 @@ class TestRunner { this.strategies = options.strategies; this.MAX_WORKERS = this.settings.MAX_WORKERS; + this.REPETITIONS = this.settings.REPETITIONS; this.MAX_TEST_ATTEMPTS = this.settings.MAX_TEST_ATTEMPTS; @@ -70,14 +71,17 @@ class TestRunner { this.allocator = options.allocator; // For each actual test path, split out - const testsXprofiles = _.flatten( + var testsXprofiles = []; + for(var cnt = 0; cnt < this.REPETITIONS; cnt++) + testsXprofiles = _.concat(testsXprofiles, _.flatten( tests.map((testLocator) => - options.profiles.map((profile) => - new Test( + options.profiles.map((profile) => { + return new Test( testLocator, profile, this.executors[profile.executor], - this.MAX_TEST_ATTEMPTS)))); + this.MAX_TEST_ATTEMPTS) + })))); if (this.settings.gatherTrends) { this.trends = { @@ -89,6 +93,7 @@ class TestRunner { this.queue = new TestQueue({ tests: testsXprofiles, workerAmount: this.MAX_WORKERS, + repetitions: this.REPETITIONS, stageTestHandler: this.stageTestHandler.bind(this), completeTestHandler: this.completeTestHandler.bind(this), @@ -211,7 +216,7 @@ class TestRunner { const profileStatement = this.profiles.map((b) => b.toString()).join(", "); const serialStatement = this.serial ? "in serial mode" : `with ${this.MAX_WORKERS} workers`; - logger.log(`Running ${this.queue.getTestAmount()} tests` + logger.log(`Running ${this.queue.getTestAmount() / this.REPETITIONS} tests` + (this.REPETITIONS > 1 ? ` repeated ${this.REPETITIONS} times` : '') + ` ${serialStatement} with [${profileStatement}]`); return this.queue.proceed(); From 108d14598aa8fd4b26a690d996b1ae02d4cf9715 Mon Sep 17 00:00:00 2001 From: brimstedt Date: Tue, 25 Sep 2018 20:46:20 +0200 Subject: [PATCH 2/5] Codestyle fixes --- src/test_runner.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test_runner.js b/src/test_runner.js index 943a0e5..e7bd027 100644 --- a/src/test_runner.js +++ b/src/test_runner.js @@ -71,17 +71,18 @@ class TestRunner { this.allocator = options.allocator; // For each actual test path, split out - var testsXprofiles = []; - for(var cnt = 0; cnt < this.REPETITIONS; cnt++) + let testsXprofiles = []; + for (let cnt = 0; cnt < this.REPETITIONS; cnt++) { testsXprofiles = _.concat(testsXprofiles, _.flatten( tests.map((testLocator) => - options.profiles.map((profile) => { - return new Test( + options.profiles.map((profile) => + new Test( testLocator, profile, this.executors[profile.executor], this.MAX_TEST_ATTEMPTS) - })))); + )))); + } if (this.settings.gatherTrends) { this.trends = { @@ -216,7 +217,8 @@ class TestRunner { const profileStatement = this.profiles.map((b) => b.toString()).join(", "); const serialStatement = this.serial ? "in serial mode" : `with ${this.MAX_WORKERS} workers`; - logger.log(`Running ${this.queue.getTestAmount() / this.REPETITIONS} tests` + (this.REPETITIONS > 1 ? ` repeated ${this.REPETITIONS} times` : '') + logger.log(`Running ${this.queue.getTestAmount() / this.REPETITIONS} tests` + + (this.REPETITIONS > 1 ? ` repeated ${this.REPETITIONS} times` : "") + ` ${serialStatement} with [${profileStatement}]`); return this.queue.proceed(); From 3651f9e71d07d61ad2c950da58d4a49ae1483be9 Mon Sep 17 00:00:00 2001 From: brimstedt Date: Wed, 26 Sep 2018 12:39:44 +0200 Subject: [PATCH 3/5] Test of repeat setting --- test/test_runner.test.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/test_runner.test.js b/test/test_runner.test.js index 481fd73..7caab05 100644 --- a/test/test_runner.test.js +++ b/test/test_runner.test.js @@ -343,6 +343,21 @@ describe("test_runner", () => { jest.runAllTimers(); }); + test("tests are repeated", (done) => { + const testRunner = new TestRunner([stubPassTest, stubPassTest], options, { + settings: { + REPETITIONS: 3 + }, + startTime: (new Date()).getTime() + }); + + testRunner.queue.proceed = jest.fn(); + testRunner.run(); + expect(testRunner.queue.proceed).toHaveBeenCalled(); + expect(testRunner.queue.tests.length).toEqual(6); + done(); + }); + test("test in serial is run successfully", (done) => { fs.mkdirSync.mockImplementation((p) => { }); @@ -666,7 +681,7 @@ describe("test_runner", () => { }); }); -function initTestRunner(tests, options, numPassedTests, numFailedTests) { +function initTestRunner(tests, options) { const testRunner = new TestRunner(tests, options, { settings: { gatherTrends: true, @@ -750,4 +765,4 @@ function stubFailTest() { pass: jest.fn(), fail: jest.fn() } -} \ No newline at end of file +} From 4e484b9dc5377b895a99c4c6c39a14949d6dd966 Mon Sep 17 00:00:00 2001 From: brimstedt Date: Sun, 30 Sep 2018 22:32:56 +0200 Subject: [PATCH 4/5] Pass repetition to plugin --- src/help.js | 2 +- src/test.js | 5 +++-- src/test_runner.js | 8 ++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/help.js b/src/help.js index 0c01eb8..5cb26a5 100644 --- a/src/help.js +++ b/src/help.js @@ -51,7 +51,7 @@ module.exports = { "category": "Strategy", "example": "testarmada-magellan-locks-resource-strategy", "visible": true, - "description": "The strategy helps magellan hold/release resourcs for test when limit resources are available." + "description": "The strategy helps magellan hold/release resources for test when limit resources are available." }, "debug": { "category": "Parallelism, Workflow and Filtering", diff --git a/src/test.js b/src/test.js index 6388b3d..53236a4 100644 --- a/src/test.js +++ b/src/test.js @@ -6,7 +6,7 @@ const TEST_STATUS_SUCCESSFUL = 3; const TEST_STATUS_SKIPPED = 4; class Test { - constructor(locator, profile, executor, maxAttempts) { + constructor(locator, profile, executor, maxAttempts, repetition) { // // note: this locator object is an instance of an object which is defined by whichever test // framework plugin is currently loaded. The implementation of locator could be almost any @@ -23,7 +23,8 @@ class Test { this.profile = profile; this.executor = executor; - + this.repetition = repetition; + this.workerIndex = -1; this.error = undefined; this.stdout = ""; diff --git a/src/test_runner.js b/src/test_runner.js index e7bd027..c90dd85 100644 --- a/src/test_runner.js +++ b/src/test_runner.js @@ -80,7 +80,8 @@ class TestRunner { testLocator, profile, this.executors[profile.executor], - this.MAX_TEST_ATTEMPTS) + this.MAX_TEST_ATTEMPTS, + cnt) )))); } @@ -499,7 +500,10 @@ class TestRunner { // executor: this.executors[test.profile.executor], // The locator object originally generated by the plugin itself - locator: test.locator + locator: test.locator, + + // Which repetition this is + repetition: test.repetition }, ports)); this.setTimeout( From fcac380a443ef5bf305a72545d4c0b5a058909de Mon Sep 17 00:00:00 2001 From: brimstedt Date: Sun, 30 Sep 2018 22:39:53 +0200 Subject: [PATCH 5/5] removed space --- src/test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test.js b/src/test.js index 53236a4..1f04920 100644 --- a/src/test.js +++ b/src/test.js @@ -24,7 +24,7 @@ class Test { this.profile = profile; this.executor = executor; this.repetition = repetition; - + this.workerIndex = -1; this.error = undefined; this.stdout = "";