Added options to specify repetitions

src/help.js

@@ -29,6 +29,12 @@ module.exports = {
       "visible": true,
       "description": "Retry tests N times (default: 3)."
     },
+    "repetitions": {
+      "category": "Parallelism, Workflow and Filtering",
+      "example": "20",
+      "visible": true,
+      "description": "Number of times to repeat each test (default: 1)."
+    },
     "test_timeout": {
       "category": "Parallelism, Workflow and Filtering",
       "example": "80000",
@@ -45,7 +51,7 @@ module.exports = {
       "category": "Strategy",
       "example": "testarmada-magellan-locks-resource-strategy",
       "visible": true,
-      "description": "The strategy helps magellan hold/release resourcs for test when limit resources are available."
+      "description": "The strategy helps magellan hold/release resources for test when limit resources are available."
     },
     "debug": {
       "category": "Parallelism, Workflow and Filtering",

src/settings.js

@@ -89,6 +89,8 @@ module.exports = {
   MAX_WORKERS: argv.serial ? 1 : parseInt(argv.max_workers) || 3,
   MAX_TEST_ATTEMPTS: parseInt(argv.max_test_attempts) || 3,
 
+  REPETITIONS: parseInt(argv.repetitions) || 1,
+
   MAX_ALLOCATION_ATTEMPTS: 120,
   WORKER_START_DELAY: 1000,
 

src/test.js

@@ -6,7 +6,7 @@ const TEST_STATUS_SUCCESSFUL = 3;
 const TEST_STATUS_SKIPPED = 4;
 
 class Test {
-  constructor(locator, profile, executor, maxAttempts) {
+  constructor(locator, profile, executor, maxAttempts, repetition) {
     //
     // note: this locator object is an instance of an object which is defined by whichever test
     // framework plugin is currently loaded. The implementation of locator could be almost any
@@ -23,6 +23,7 @@ class Test {
 
     this.profile = profile;
     this.executor = executor;
+    this.repetition = repetition;
 
     this.workerIndex = -1;
     this.error = undefined;

src/test_runner.js

@@ -53,6 +53,7 @@ class TestRunner {
     this.strategies = options.strategies;
 
     this.MAX_WORKERS = this.settings.MAX_WORKERS;
+    this.REPETITIONS = this.settings.REPETITIONS;
 
     this.MAX_TEST_ATTEMPTS = this.settings.MAX_TEST_ATTEMPTS;
 
@@ -70,14 +71,19 @@ class TestRunner {
 
     this.allocator = options.allocator;
     // For each actual test path, split out
-    const testsXprofiles = _.flatten(
+    let testsXprofiles = [];
+    for (let cnt = 0; cnt < this.REPETITIONS; cnt++) {
+      testsXprofiles = _.concat(testsXprofiles, _.flatten(
       tests.map((testLocator) =>
         options.profiles.map((profile) =>
           new Test(
             testLocator,
             profile,
             this.executors[profile.executor],
-            this.MAX_TEST_ATTEMPTS))));
+            this.MAX_TEST_ATTEMPTS,
+            cnt)
+        ))));
+    }
 
     if (this.settings.gatherTrends) {
       this.trends = {
@@ -89,6 +95,7 @@ class TestRunner {
     this.queue = new TestQueue({
       tests: testsXprofiles,
       workerAmount: this.MAX_WORKERS,
+      repetitions: this.REPETITIONS,
 
       stageTestHandler: this.stageTestHandler.bind(this),
       completeTestHandler: this.completeTestHandler.bind(this),
@@ -211,7 +218,8 @@ class TestRunner {
     const profileStatement = this.profiles.map((b) => b.toString()).join(", ");
     const serialStatement = this.serial ? "in serial mode" : `with ${this.MAX_WORKERS} workers`;
 
-    logger.log(`Running ${this.queue.getTestAmount()} tests`
+    logger.log(`Running ${this.queue.getTestAmount() / this.REPETITIONS} tests`
+      + (this.REPETITIONS > 1 ? ` repeated ${this.REPETITIONS} times` : "")
       + ` ${serialStatement} with [${profileStatement}]`);
 
     return this.queue.proceed();
@@ -492,7 +500,10 @@ class TestRunner {
           // executor: this.executors[test.profile.executor],
 
           // The locator object originally generated by the plugin itself
-          locator: test.locator
+          locator: test.locator,
+
+          // Which repetition this is
+          repetition: test.repetition
         }, ports));
 
         this.setTimeout(

test/test_runner.test.js

@@ -343,6 +343,21 @@ describe("test_runner", () => {
       jest.runAllTimers();
     });
 
+    test("tests are repeated", (done) => {
+      const testRunner = new TestRunner([stubPassTest, stubPassTest], options, {
+        settings: {
+          REPETITIONS: 3
+        },
+        startTime: (new Date()).getTime()
+      });
+
+      testRunner.queue.proceed = jest.fn();
+      testRunner.run();
+      expect(testRunner.queue.proceed).toHaveBeenCalled();
+      expect(testRunner.queue.tests.length).toEqual(6);
+      done();
+    });
+
     test("test in serial is run successfully", (done) => {
       fs.mkdirSync.mockImplementation((p) => { });
 
@@ -666,7 +681,7 @@ describe("test_runner", () => {
   });
 });
 
-function initTestRunner(tests, options, numPassedTests, numFailedTests) {
+function initTestRunner(tests, options) {
   const testRunner = new TestRunner(tests, options, {
     settings: {
       gatherTrends: true,
@@ -750,4 +765,4 @@ function stubFailTest() {
     pass: jest.fn(),
     fail: jest.fn()
   }
-}
+}