test_runner: feat add coverage threshold for tests

Author: pulkit-30

index.mjs

@@ -0,0 +1,11 @@
+import { spec } from 'node:test/reporters';
+import { run } from 'node:test';
+import process from 'node:process';
+
+run({ files: ['./index.test.mjs'], coverage: {
+    lines: 100,
+    branches: 100,
+    functions: 100,
+} })
+  .compose(spec)
+  .pipe(process.stdout);

index.test.mjs

@@ -0,0 +1,6 @@
+import assert from 'node:assert';
+import test from 'node:test';
+
+test('hello test', () => {
+  // throw new Error("error");
+});

lib/internal/test_runner/coverage.js

@@ -31,6 +31,12 @@ const kIgnoreRegex = /\/\* node:coverage ignore next (?<count>\d+ )?\*\//;
 const kLineEndingRegex = /\r?\n$/u;
 const kLineSplitRegex = /(?<=\r?\n)/u;
 const kStatusRegex = /\/\* node:coverage (?<status>enable|disable) \*\//;
+const kDefaultMinimumThreshold = {
+  __proto__: null,
+  lines: 0,
+  functions: 0,
+  branches: 0,
+};
 
 class CoverageLine {
   #covered;
@@ -63,10 +69,11 @@ class CoverageLine {
 }
 
 class TestCoverage {
-  constructor(coverageDirectory, originalCoverageDirectory, workingDirectory) {
+  constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, minimumThreshold) {
     this.coverageDirectory = coverageDirectory;
     this.originalCoverageDirectory = originalCoverageDirectory;
     this.workingDirectory = workingDirectory;
+    this.minimumThreshold = minimumThreshold || kDefaultMinimumThreshold;
   }
 
   summary() {
@@ -260,6 +267,12 @@ class TestCoverage {
     );
     coverageSummary.files.sort(sortCoverageFiles);
 
+    coverageSummary.threshold = {
+      __proto__: null,
+      lines: doesThresholdPass(this.minimumThreshold.lines, coverageSummary.totals.coveredLinePercent),
+      functions: doesThresholdPass(this.minimumThreshold.functions, coverageSummary.totals.coveredFunctionPercent),
+      branches: doesThresholdPass(this.minimumThreshold.branches, coverageSummary.totals.coveredBranchPercent),
+    };
     return coverageSummary;
   }
 
@@ -299,7 +312,7 @@ function sortCoverageFiles(a, b) {
   return StringPrototypeLocaleCompare(a.path, b.path);
 }
 
-function setupCoverage() {
+function setupCoverage(minimumThreshold = null) {
   let originalCoverageDirectory = process.env.NODE_V8_COVERAGE;
   const cwd = process.cwd();
 
@@ -323,7 +336,7 @@ function setupCoverage() {
   // child processes.
   process.env.NODE_V8_COVERAGE = coverageDirectory;
 
-  return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd);
+  return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd, minimumThreshold);
 }
 
 function mapRangeToLines(range, lines) {
@@ -538,4 +551,8 @@ function doesRangeContainOtherRange(range, otherRange) {
          range.endOffset >= otherRange.endOffset;
 }
 
+function doesThresholdPass(compareValue, actualValue) {
+  return compareValue ? actualValue >= compareValue : true;
+}
+
 module.exports = { setupCoverage, TestCoverage };

lib/internal/test_runner/harness.js

@@ -1,9 +1,12 @@
 'use strict';
 const {
+  ArrayPrototypeFilter,
   ArrayPrototypeForEach,
   FunctionPrototypeBind,
   PromiseResolve,
   SafeMap,
+  StringPrototypeSplit,
+  StringPrototypeStartsWith,
 } = primordials;
 const { getCallerLocation } = internalBinding('util');
 const {
@@ -78,14 +81,20 @@ function createProcessEventHandler(eventName, rootTest) {
 }
 
 function configureCoverage(rootTest, globalOptions) {
-  if (!globalOptions.coverage) {
-    return null;
+  let coverageThreshold = rootTest.coverage;
+  if (!coverageThreshold) {
+    if (!globalOptions.coverage) {
+      return null;
+    }
+    const { 1: value } = StringPrototypeSplit(
+      ArrayPrototypeFilter(process.execArgv, (arg) =>
+        StringPrototypeStartsWith(arg, '--experimental-test-coverage')), '=');
+    coverageThreshold = { __proto__: null, lines: value, functions: value, branches: value };
   }
-
   const { setupCoverage } = require('internal/test_runner/coverage');
 
   try {
-    return setupCoverage();
+    return setupCoverage(coverageThreshold);
   } catch (err) {
     const msg = `Warning: Code coverage could not be enabled. ${err}`;
 

lib/internal/test_runner/runner.js

@@ -439,7 +439,7 @@ function run(options = kEmptyObject) {
   validateObject(options, 'options');
 
   let { testNamePatterns, shard } = options;
-  const { concurrency, timeout, signal, files, inspectPort, watch, setup, only } = options;
+  const { concurrency, timeout, signal, files, inspectPort, watch, setup, only, coverage } = options;
 
   if (files != null) {
     validateArray(files, 'options.files');
@@ -486,7 +486,15 @@ function run(options = kEmptyObject) {
     });
   }
 
-  const root = createTestTree({ __proto__: null, concurrency, timeout, signal });
+  if (coverage != null) {
+    if (typeof coverage === 'boolean') {
+      validateBoolean(coverage, 'options.coverage');
+    } else {
+      validateObject(coverage, 'options.coverage');
+    }
+  }
+
+  const root = createTestTree({ __proto__: null, concurrency, timeout, signal, coverage });
   if (process.env.NODE_TEST_CONTEXT !== undefined) {
     return root.reporter;
   }

lib/internal/test_runner/test.js

@@ -1,5 +1,7 @@
 'use strict';
 const {
+  ArrayPrototypeFilter,
+  ArrayPrototypeJoin,
   ArrayPrototypePush,
   ArrayPrototypePushApply,
   ArrayPrototypeReduce,
@@ -10,6 +12,8 @@ const {
   FunctionPrototype,
   MathMax,
   Number,
+  ObjectDefineProperty,
+  ObjectKeys,
   ObjectSeal,
   PromisePrototypeThen,
   PromiseResolve,
@@ -21,7 +25,6 @@ const {
   SafePromiseAll,
   SafePromiseRace,
   SymbolDispose,
-  ObjectDefineProperty,
   Symbol,
 } = primordials;
 const { getCallerLocation } = internalBinding('util');
@@ -49,6 +52,7 @@ const {
   once: runOnce,
 } = require('internal/util');
 const { isPromise } = require('internal/util/types');
+const console = require('internal/console/global');
 const {
   validateAbortSignal,
   validateNumber,
@@ -209,7 +213,7 @@ class Test extends AsyncResource {
     super('Test');
 
     let { fn, name, parent, skip } = options;
-    const { concurrency, loc, only, timeout, todo, signal } = options;
+    const { concurrency, loc, only, timeout, todo, signal, coverage } = options;
 
     if (typeof fn !== 'function') {
       fn = noop;
@@ -239,6 +243,7 @@ class Test extends AsyncResource {
         beforeEach: [],
         afterEach: [],
       };
+      this.coverage = coverage;
     } else {
       const nesting = parent.parent === null ? parent.nesting :
         parent.nesting + 1;
@@ -258,6 +263,7 @@ class Test extends AsyncResource {
         beforeEach: ArrayPrototypeSlice(parent.hooks.beforeEach),
         afterEach: ArrayPrototypeSlice(parent.hooks.afterEach),
       };
+      this.coverage = parent.coverage;
     }
 
     switch (typeof concurrency) {
@@ -670,7 +676,9 @@ class Test extends AsyncResource {
       // postRun() method is called when the process is getting ready to exit.
       // This helps catch any asynchronous activity that occurs after the tests
       // have finished executing.
-      this.postRun();
+      try {
+        this.postRun();
+      } catch { /* ignore error */ }
     }
   }
 
@@ -754,6 +762,12 @@ class Test extends AsyncResource {
 
       if (coverage) {
         reporter.coverage(nesting, loc, coverage);
+        const failedCoverage = ArrayPrototypeFilter(
+          ObjectKeys(coverage.threshold), (key) => !coverage.threshold[key]);
+        if (failedCoverage.length > 0) {
+          console.error(`test coverage failed for ${ArrayPrototypeJoin(failedCoverage, ', ')}`);
+          process.exit(1);
+        }
       }
 
       reporter.end();

lib/internal/test_runner/utils.js

@@ -3,17 +3,19 @@ const {
   ArrayPrototypeJoin,
   ArrayPrototypeMap,
   ArrayPrototypeFlatMap,
+  ArrayPrototypeForEach,
   ArrayPrototypePush,
   ArrayPrototypeReduce,
   ObjectGetOwnPropertyDescriptor,
+  ObjectKeys,
   MathFloor,
   MathMax,
   MathMin,
   NumberPrototypeToFixed,
-  SafePromiseAllReturnArrayLike,
   RegExp,
   RegExpPrototypeExec,
   SafeMap,
+  SafePromiseAllReturnArrayLike,
   StringPrototypePadStart,
   StringPrototypePadEnd,
   StringPrototypeRepeat,
@@ -316,6 +318,7 @@ const kSeparator = ' | ';
 
 function getCoverageReport(pad, summary, symbol, color, table) {
   const prefix = `${pad}${symbol}`;
+  color ||= white;
   let report = `${color}${prefix}start of coverage report\n`;
 
   let filePadLength;
@@ -405,6 +408,12 @@ function getCoverageReport(pad, summary, symbol, color, table) {
             `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumnsKeys, (columnKey, j) => getCell(NumberPrototypeToFixed(summary.totals[columnKey], 2), columnPadLengths[j], StringPrototypePadStart, false, summary.totals[columnKey])), kSeparator)} |\n`;
   if (table) report += addTableLine(prefix, tableWidth);
 
+  ArrayPrototypeForEach(ObjectKeys(summary.threshold), (key) => {
+    if (!summary.threshold[key]) {
+      report += `${red}${prefix}coverage threshold for ${key} not met\n${color}`;
+    }
+  });
+
   report += `${prefix}end of coverage report\n`;
   if (color) {
     report += white;