Merge pull request #5809 from NomicFoundation/feat/solidity-test-repo…

…rter

feat: replace spec with a custom hardhat solidity test reporter

v-next/hardhat-errors/src/descriptors.ts

@@ -716,6 +716,14 @@ This might be caused by using hardhat_reset and loadFixture calls in a testcase.
       websiteTitle: `Build info not found for contract`,
       websiteDescription: `Build info not found for contract while compiling Solidity test contracts.`,
     },
+    RUNNER_TIMEOUT: {
+      number: 1001,
+      messageTemplate: `Runner timed out after {duration} ms.
+
+Remaining test suites: {suites}`,
+      websiteTitle: `Runner timed out`,
+      websiteDescription: `Runner timed out while running Solidity tests.`,
+    },
   },
   ETHERS: {
     METHOD_NOT_IMPLEMENTED: {

v-next/hardhat/src/internal/builtin-plugins/solidity-test/formatters.ts

@@ -0,0 +1,5 @@
+import type { ArtifactId } from "@ignored/edr";
+
+export function formatArtifactId(artifactId: ArtifactId): string {
+  return `${artifactId.source}:${artifactId.name} (v${artifactId.solcVersion})`;
+}

v-next/hardhat/src/internal/builtin-plugins/solidity-test/helpers.ts

@@ -1,55 +1,8 @@
 import type { ArtifactsManager } from "../../../types/artifacts.js";
-import type {
-  ArtifactId,
-  SuiteResult,
-  Artifact,
-  SolidityTestRunnerConfigArgs,
-  TestResult,
-} from "@ignored/edr";
+import type { ArtifactId, Artifact } from "@ignored/edr";
 
-import { runSolidityTests } from "@ignored/edr";
 import { HardhatError } from "@ignored/hardhat-vnext-errors";
 
-/**
- * Run all the given solidity tests and returns the whole results after finishing.
- *
- * This function is a direct port of the example v2 integration in the
- * EDR repo (see  https://github.com/NomicFoundation/edr/blob/feat/solidity-tests/js/helpers/src/index.ts).
- * The signature of the function should be considered a draft and may change in the future.
- *
- * TODO: Reconsider the signature and feedback to EDR team.
- */
-export async function runAllSolidityTests(
-  artifacts: Artifact[],
-  testSuites: ArtifactId[],
-  configArgs: SolidityTestRunnerConfigArgs,
-  testResultCallback: (
-    suiteResult: SuiteResult,
-    testResult: TestResult,
-  ) => void = () => {},
-): Promise<SuiteResult[]> {
-  return new Promise((resolve, reject) => {
-    const resultsFromCallback: SuiteResult[] = [];
-
-    runSolidityTests(
-      artifacts,
-      testSuites,
-      configArgs,
-      (suiteResult: SuiteResult) => {
-        for (const testResult of suiteResult.testResults) {
-          testResultCallback(suiteResult, testResult);
-        }
-
-        resultsFromCallback.push(suiteResult);
-        if (resultsFromCallback.length === testSuites.length) {
-          resolve(resultsFromCallback);
-        }
-      },
-      reject,
-    );
-  });
-}
-
 export async function buildSolidityTestsInput(
   hardhatArtifacts: ArtifactsManager,
   isTestArtifact: (artifact: Artifact) => boolean = () => true,

v-next/hardhat/src/internal/builtin-plugins/solidity-test/index.ts

@@ -1,12 +1,20 @@
 import type { HardhatPlugin } from "../../../types/plugins.js";
 
+import { ArgumentType } from "../../../types/arguments.js";
 import { task } from "../../core/config.js";
 
 const hardhatPlugin: HardhatPlugin = {
   id: "builtin:solidity-test",
   tasks: [
     task(["test:solidity"], "Run the Solidity tests")
       .setAction(import.meta.resolve("./task-action.js"))
+      .addOption({
+        name: "timeout",
+        description:
+          "The maximum time in milliseconds to wait for all the test suites to finish",
+        type: ArgumentType.INT,
+        defaultValue: 60 * 60 * 1000,
+      })
       .build(),
   ],
 };

v-next/hardhat/src/internal/builtin-plugins/solidity-test/reporter.ts

@@ -0,0 +1,60 @@
+import type {
+  TestEventSource,
+  TestReporterResult,
+  TestStatus,
+} from "./types.js";
+
+import chalk from "chalk";
+
+/**
+ * This is a solidity test reporter. It is intended to be composed with the
+ * solidity test runner's test stream. It was based on the hardhat node test
+ * reporter's design.
+ */
+export async function* testReporter(
+  source: TestEventSource,
+): TestReporterResult {
+  let testResultCount = 0;
+  let successCount = 0;
+  let failureCount = 0;
+  let skippedCount = 0;
+
+  for await (const event of source) {
+    switch (event.type) {
+      case "suite:result": {
+        const { data: suiteResult } = event;
+        for (const testResult of suiteResult.testResults) {
+          testResultCount++;
+          let name = suiteResult.id.name + " | " + testResult.name;
+          if ("runs" in testResult?.kind) {
+            name += ` (${testResult.kind.runs} runs)`;
+          }
+          const status: TestStatus = testResult.status;
+          switch (status) {
+            case "Success": {
+              yield chalk.green(`✔ ${name}`);
+              successCount++;
+              break;
+            }
+            case "Failure": {
+              yield chalk.red(`✖ ${name}`);
+              failureCount++;
+              break;
+            }
+            case "Skipped": {
+              yield chalk.yellow(`⚠️ ${name}`);
+              skippedCount++;
+              break;
+            }
+          }
+          yield "\n";
+        }
+        break;
+      }
+    }
+  }
+
+  yield "\n";
+  yield `${testResultCount} tests found, ${successCount} passed, ${failureCount} failed, ${skippedCount} skipped`;
+  yield "\n";
+}

v-next/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts

@@ -0,0 +1,92 @@
+import type { TestEvent, TestsStream } from "./types.js";
+import type {
+  ArtifactId,
+  Artifact,
+  SolidityTestRunnerConfigArgs,
+} from "@ignored/edr";
+
+import { Readable } from "node:stream";
+
+import { runSolidityTests } from "@ignored/edr";
+import { HardhatError } from "@ignored/hardhat-vnext-errors";
+
+import { formatArtifactId } from "./formatters.js";
+
+export interface RunOptions {
+  /**
+   * The maximum time in milliseconds to wait for all the test suites to finish.
+   *
+   * If not provided, the default is 1 hour.
+   */
+  timeout?: number;
+}
+
+/**
+ * Run all the given solidity tests and returns the stream of results.
+ *
+ * It returns a Readable stream that emits the test events similarly to how the
+ * node test runner does it.
+ *
+ * The stream is closed when all the test suites have been run.
+ *
+ * This function, initially, was a direct port of the example v2 integration in
+ * the EDR repo (see  https://github.com/NomicFoundation/edr/blob/feat/solidity-tests/js/helpers/src/index.ts).
+ *
+ * Despite the changes, the signature of the function should still be considered
+ * a draft that may change in the future.
+ *
+ * TODO: Once the signature is finalised, give feedback to the EDR team.
+ */
+export function run(
+  artifacts: Artifact[],
+  testSuiteIds: ArtifactId[],
+  configArgs: SolidityTestRunnerConfigArgs,
+  options?: RunOptions,
+): TestsStream {
+  const stream = new ReadableStream<TestEvent>({
+    start(controller) {
+      if (testSuiteIds.length === 0) {
+        controller.close();
+        return;
+      }
+
+      const remainingSuites = new Set(testSuiteIds.map(formatArtifactId));
+
+      // NOTE: The timeout prevents the situation in which the stream is never
+      // closed. This can happen if we receive fewer suite results than the
+      // number of test suites. The timeout is set to 1 hour.
+      const duration = options?.timeout ?? 60 * 60 * 1000;
+      const timeout = setTimeout(() => {
+        controller.error(
+          new HardhatError(HardhatError.ERRORS.SOLIDITY_TESTS.RUNNER_TIMEOUT, {
+            duration,
+            suites: Array.from(remainingSuites).join(", "),
+          }),
+        );
+      }, duration);
+
+      runSolidityTests(
+        artifacts,
+        testSuiteIds,
+        configArgs,
+        (suiteResult) => {
+          controller.enqueue({
+            type: "suite:result",
+            data: suiteResult,
+          });
+          remainingSuites.delete(formatArtifactId(suiteResult.id));
+          if (remainingSuites.size === 0) {
+            clearTimeout(timeout);
+            controller.close();
+          }
+        },
+        (error) => {
+          clearTimeout(timeout);
+          controller.error(error);
+        },
+      );
+    },
+  });
+
+  return Readable.from(stream);
+}

v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts

@@ -1,21 +1,18 @@
+import type { RunOptions } from "./runner.js";
+import type { TestEvent } from "./types.js";
 import type { NewTaskActionFunction } from "../../../types/tasks.js";
 
-import { spec } from "node:test/reporters";
+import { finished } from "node:stream/promises";
 
-import { buildSolidityTestsInput, runAllSolidityTests } from "./helpers.js";
+import { buildSolidityTestsInput } from "./helpers.js";
+import { testReporter } from "./reporter.js";
+import { run } from "./runner.js";
 
-const runSolidityTests: NewTaskActionFunction = async (_arguments, hre) => {
+const runSolidityTests: NewTaskActionFunction = async ({ timeout }, hre) => {
   await hre.tasks.getTask("compile").run({ quiet: false });
 
   console.log("\nRunning Solidity tests...\n");
 
-  const specReporter = new spec();
-
-  specReporter.pipe(process.stdout);
-
-  let totalTests = 0;
-  let failedTests = 0;
-
   const { artifacts, testSuiteIds } = await buildSolidityTestsInput(
     hre.artifacts,
     (artifact) => {
@@ -34,35 +31,34 @@ const runSolidityTests: NewTaskActionFunction = async (_arguments, hre) => {
     projectRoot: hre.config.paths.root,
   };
 
-  await runAllSolidityTests(
-    artifacts,
-    testSuiteIds,
-    config,
-    (suiteResult, testResult) => {
-      let name = suiteResult.id.name + " | " + testResult.name;
-      if ("runs" in testResult?.kind) {
-        name += ` (${testResult.kind.runs} runs)`;
-      }
-
-      totalTests++;
+  let includesFailures = false;
+  let includesErrors = false;
 
-      const failed = testResult.status === "Failure";
-      if (failed) {
-        failedTests++;
-      }
+  const options: RunOptions = { timeout };
 
-      specReporter.write({
-        type: failed ? "test:fail" : "test:pass",
-        data: {
-          name,
-        },
-      });
-    },
-  );
+  const runStream = run(artifacts, testSuiteIds, config, options);
 
-  console.log(`\n${totalTests} tests found, ${failedTests} failed`);
+  runStream
+    .on("data", (event: TestEvent) => {
+      if (event.type === "suite:result") {
+        if (event.data.testResults.some(({ status }) => status === "Failure")) {
+          includesFailures = true;
+        }
+      }
+    })
+    .compose(testReporter)
+    .pipe(process.stdout);
+
+  // NOTE: We're awaiting the original run stream to finish instead of the
+  // composed reporter stream to catch any errors produced by the runner.
+  try {
+    await finished(runStream);
+  } catch (error) {
+    console.error(error);
+    includesErrors = true;
+  }
 
-  if (failedTests > 0) {
+  if (includesFailures || includesErrors) {
     process.exitCode = 1;
     return;
   }

v-next/hardhat/src/internal/builtin-plugins/solidity-test/types.ts

@@ -0,0 +1,17 @@
+import type { SuiteResult } from "@ignored/edr";
+import type { Readable } from "node:stream";
+
+export type TestStatus = "Success" | "Failure" | "Skipped";
+
+export type TestsStream = Readable;
+
+// NOTE: The interface can be turned into a type and extended with more event types as needed.
+export interface TestEvent {
+  type: "suite:result";
+  data: SuiteResult;
+}
+
+export type TestEventSource = AsyncGenerator<TestEvent, void>;
+export type TestReporterResult = AsyncGenerator<string, void>;
+
+export type TestReporter = (source: TestEventSource) => TestReporterResult;