feat: add a configurable timeout for solidity tests execution

v-next/hardhat-errors/src/descriptors.ts

@@ -701,6 +701,14 @@ This might be caused by using hardhat_reset and loadFixture calls in a testcase.
       websiteTitle: `Build info not found for contract`,
       websiteDescription: `Build info not found for contract while compiling Solidity test contracts.`,
     },
+    RUNNER_TIMEOUT: {
+      number: 1001,
+      messageTemplate: `Runner timed out after {duration} ms.
+
+Remaining test suites: {suites}`,
+      websiteTitle: `Runner timed out`,
+      websiteDescription: `Runner timed out while running Solidity tests.`,
+    },
   },
   ETHERS: {
     METHOD_NOT_IMPLEMENTED: {

v-next/hardhat/src/internal/builtin-plugins/solidity-test/formatters.ts

@@ -0,0 +1,5 @@
+import type { ArtifactId } from "@ignored/edr";
+
+export function formatArtifactId(artifactId: ArtifactId): string {
+  return `${artifactId.source}:${artifactId.name} (v${artifactId.solcVersion})`;
+}

v-next/hardhat/src/internal/builtin-plugins/solidity-test/index.ts

@@ -1,12 +1,20 @@
 import type { HardhatPlugin } from "../../../types/plugins.js";
 
+import { ArgumentType } from "../../../types/arguments.js";
 import { task } from "../../core/config.js";
 
 const hardhatPlugin: HardhatPlugin = {
   id: "builtin:solidity-test",
   tasks: [
     task(["test:solidity"], "Run the Solidity tests")
       .setAction(import.meta.resolve("./task-action.js"))
+      .addOption({
+        name: "timeout",
+        description:
+          "The maximum time in milliseconds to wait for all the test suites to finish",
+        type: ArgumentType.INT,
+        defaultValue: 60 * 60 * 1000,
+      })
       .build(),
   ],
 };

v-next/hardhat/src/internal/builtin-plugins/solidity-test/reporter.ts

@@ -51,7 +51,7 @@ export async function* testReporter(
         }
         break;
       }
-      case "test:complete": {
+      case "run:complete": {
         yield "\n";
         yield `${testResultCount} tests found, ${successCount} passed, ${failureCount} failed, ${skippedCount} skipped`;
         yield "\n";

v-next/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts

@@ -8,13 +8,27 @@ import type {
 import { Readable } from "node:stream";
 
 import { runSolidityTests } from "@ignored/edr";
+import { HardhatError } from "@ignored/hardhat-vnext-errors";
+
+import { formatArtifactId } from "./formatters.js";
+
+export interface RunOptions {
+  /**
+   * The maximum time in milliseconds to wait for all the test suites to finish.
+   *
+   * If not provided, the default is 1 hour.
+   */
+  timeout?: number;
+}
 
 /**
  * Run all the given solidity tests and returns the stream of results.
  *
  * It returns a Readable stream that emits the test events similarly to how the
  * node test runner does it.
  *
+ * The stream is closed when all the test suites have been run.
+ *
  * This function, initially, was a direct port of the example v2 integration in
  * the EDR repo (see  https://github.com/NomicFoundation/edr/blob/feat/solidity-tests/js/helpers/src/index.ts).
  *
@@ -27,24 +41,39 @@ export function run(
   artifacts: Artifact[],
   testSuiteIds: ArtifactId[],
   configArgs: SolidityTestRunnerConfigArgs,
+  options?: RunOptions,
 ): TestsStream {
-  let resultCount = 0;
-
   const stream = new ReadableStream<TestEvent>({
     start(controller) {
+      const remainingSuites = new Set(testSuiteIds.map(formatArtifactId));
+
+      // NOTE: The timeout prevents the situation in which the stream is never
+      // closed. This can happen if we receive fewer suite results than the
+      // number of test suites. The timeout is set to 1 hour.
+      const duration = options?.timeout ?? 60 * 60 * 1000;
+      const timeout = setTimeout(() => {
+        controller.error(
+          new HardhatError(HardhatError.ERRORS.SOLIDITY_TESTS.RUNNER_TIMEOUT, {
+            duration,
+            suites: Array.from(remainingSuites).join(", "),
+          }),
+        );
+      }, duration);
+
       runSolidityTests(
         artifacts,
         testSuiteIds,
         configArgs,
         (suiteResult) => {
-          resultCount++;
           controller.enqueue({
             type: "suite:result",
             data: suiteResult,
           });
-          if (resultCount === testSuiteIds.length) {
+          remainingSuites.delete(formatArtifactId(suiteResult.id));
+          if (remainingSuites.size === 0) {
+            clearTimeout(timeout);
             controller.enqueue({
-              type: "test:complete",
+              type: "run:complete",
               data: undefined,
             });
             controller.close();

v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts

@@ -7,7 +7,7 @@ import { getArtifactsAndTestSuiteIds } from "./helpers.js";
 import { testReporter } from "./reporter.js";
 import { run } from "./runner.js";
 
-const runSolidityTests: NewTaskActionFunction = async (_arguments, hre) => {
+const runSolidityTests: NewTaskActionFunction = async ({ timeout }, hre) => {
   await hre.tasks.getTask("compile").run({ quiet: false });
 
   console.log("\nRunning Solidity tests...\n");
@@ -21,25 +21,33 @@ const runSolidityTests: NewTaskActionFunction = async (_arguments, hre) => {
   };
 
   let includesFailures = false;
+  let includesErrors = false;
 
-  const reporterStream = run(artifacts, testSuiteIds, config)
+  const options = { timeout };
+
+  const runStream = run(artifacts, testSuiteIds, config, options);
+
+  runStream
     .on("data", (event: TestEvent) => {
       if (event.type === "suite:result") {
         if (event.data.testResults.some(({ status }) => status === "Failure")) {
           includesFailures = true;
         }
       }
     })
-    .compose(testReporter);
-
-  reporterStream.pipe(process.stdout);
-
-  // NOTE: If the stream does not end (e.g. if EDR does not report on all the
-  // test suites), this promise is never resolved but the process will happily
-  // exit with a zero exit code without continuing past this point 😕
-  await finished(reporterStream);
+    .compose(testReporter)
+    .pipe(process.stdout);
+
+  // NOTE: We're awaiting the original run stream to finish instead of the
+  // composed reporter stream to catch any errors produced by the runner.
+  try {
+    await finished(runStream);
+  } catch (error) {
+    console.error(error);
+    includesErrors = true;
+  }
 
-  if (includesFailures) {
+  if (includesFailures || includesErrors) {
     process.exitCode = 1;
     return;
   }

v-next/hardhat/src/internal/builtin-plugins/solidity-test/types.ts

@@ -7,7 +7,7 @@ export type TestsStream = Readable;
 
 export type TestEvent =
   | { type: "suite:result"; data: SuiteResult }
-  | { type: "test:complete"; data: undefined };
+  | { type: "run:complete"; data: undefined };
 
 export type TestEventSource = AsyncGenerator<TestEvent, void>;
 export type TestReporterResult = AsyncGenerator<string, void>;