test_runner: added coverage threshold support for tests

Author: pulkit-30

lib/internal/test_runner/coverage.js

@@ -31,6 +31,12 @@ const kIgnoreRegex = /\/\* node:coverage ignore next (?<count>\d+ )?\*\//;
 const kLineEndingRegex = /\r?\n$/u;
 const kLineSplitRegex = /(?<=\r?\n)/u;
 const kStatusRegex = /\/\* node:coverage (?<status>enable|disable) \*\//;
+const kDefaultMinimumCoverage = {
+  __proto__: null,
+  line: 0,
+  function: 0,
+  branch: 0,
+};
 
 class CoverageLine {
   #covered;
@@ -63,10 +69,11 @@ class CoverageLine {
 }
 
 class TestCoverage {
-  constructor(coverageDirectory, originalCoverageDirectory, workingDirectory) {
+  constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, minimumCoverage) {
     this.coverageDirectory = coverageDirectory;
     this.originalCoverageDirectory = originalCoverageDirectory;
     this.workingDirectory = workingDirectory;
+    this.minimumCoverage = minimumCoverage || kDefaultMinimumCoverage;
   }
 
   summary() {
@@ -260,6 +267,24 @@ class TestCoverage {
     );
     coverageSummary.files.sort(sortCoverageFiles);
 
+    coverageSummary.minimumCoverage = {
+      __proto__: null,
+      line: {
+        status: doesCoveragePass(this.minimumCoverage.line, coverageSummary.totals.coveredLinePercent),
+        expected: this.minimumCoverage.line,
+        actual: coverageSummary.totals.coveredLinePercent,
+      },
+      function: {
+        status: doesCoveragePass(this.minimumCoverage.function, coverageSummary.totals.coveredFunctionPercent),
+        expected:this.minimumCoverage.function,
+        actual: coverageSummary.totals.coveredFunctionPercent
+      },
+      branch: {
+        status: doesCoveragePass(this.minimumCoverage.branch, coverageSummary.totals.coveredBranchPercent),
+        expected: this.minimumCoverage.branch,
+        actual: coverageSummary.totals.coveredBranchPercent
+      },
+    };
     return coverageSummary;
   }
 
@@ -299,7 +324,7 @@ function sortCoverageFiles(a, b) {
   return StringPrototypeLocaleCompare(a.path, b.path);
 }
 
-function setupCoverage() {
+function setupCoverage(minimumCoverage = null) {
   let originalCoverageDirectory = process.env.NODE_V8_COVERAGE;
   const cwd = process.cwd();
 
@@ -323,7 +348,7 @@ function setupCoverage() {
   // child processes.
   process.env.NODE_V8_COVERAGE = coverageDirectory;
 
-  return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd);
+  return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd, minimumCoverage);
 }
 
 function mapRangeToLines(range, lines) {
@@ -538,4 +563,8 @@ function doesRangeContainOtherRange(range, otherRange) {
          range.endOffset >= otherRange.endOffset;
 }
 
+function doesCoveragePass(compareValue, actualValue) {
+  return compareValue ? actualValue >= compareValue : true;
+}
+
 module.exports = { setupCoverage, TestCoverage };

lib/internal/test_runner/harness.js

@@ -1,9 +1,12 @@
 'use strict';
 const {
+  ArrayPrototypeFilter,
   ArrayPrototypeForEach,
   FunctionPrototypeBind,
   PromiseResolve,
   SafeMap,
+  StringPrototypeSplit,
+  StringPrototypeStartsWith,
 } = primordials;
 const { getCallerLocation } = internalBinding('util');
 const {
@@ -78,14 +81,22 @@ function createProcessEventHandler(eventName, rootTest) {
 }
 
 function configureCoverage(rootTest, globalOptions) {
-  if (!globalOptions.coverage) {
-    return null;
-  }
+  let minimumCoverage = rootTest.coverage;
+  if (!minimumCoverage) {
+    if (!globalOptions.coverage) {
+      return null;
+    }
 
+    const { 1: value } = StringPrototypeSplit(
+      ArrayPrototypeFilter(process.execArgv, (arg) =>
+        StringPrototypeStartsWith(arg, '--experimental-test-coverage')), '=');
+
+    minimumCoverage = { __proto__: null, line: value, function: value, branch: value };
+  }
   const { setupCoverage } = require('internal/test_runner/coverage');
 
   try {
-    return setupCoverage();
+    return setupCoverage(minimumCoverage);
   } catch (err) {
     const msg = `Warning: Code coverage could not be enabled. ${err}`;
 

lib/internal/test_runner/runner.js

@@ -439,7 +439,7 @@ function run(options = kEmptyObject) {
   validateObject(options, 'options');
 
   let { testNamePatterns, shard } = options;
-  const { concurrency, timeout, signal, files, inspectPort, watch, setup, only } = options;
+  const { concurrency, timeout, signal, files, inspectPort, watch, setup, only, coverage } = options;
 
   if (files != null) {
     validateArray(files, 'options.files');
@@ -486,7 +486,15 @@ function run(options = kEmptyObject) {
     });
   }
 
-  const root = createTestTree({ __proto__: null, concurrency, timeout, signal });
+  if (coverage != null) {
+    if (typeof coverage === 'boolean') {
+      validateBoolean(coverage, 'options.coverage');
+    } else {
+      validateObject(coverage, 'options.coverage');
+    }
+  }
+
+  const root = createTestTree({ __proto__: null, concurrency, timeout, signal, coverage });
   if (process.env.NODE_TEST_CONTEXT !== undefined) {
     return root.reporter;
   }

lib/internal/test_runner/test.js

@@ -1,5 +1,6 @@
 'use strict';
 const {
+  ArrayPrototypeFilter,
   ArrayPrototypePush,
   ArrayPrototypePushApply,
   ArrayPrototypeReduce,
@@ -10,6 +11,8 @@ const {
   FunctionPrototype,
   MathMax,
   Number,
+  ObjectDefineProperty,
+  ObjectKeys,
   ObjectSeal,
   PromisePrototypeThen,
   PromiseResolve,
@@ -21,7 +24,6 @@ const {
   SafePromiseAll,
   SafePromiseRace,
   SymbolDispose,
-  ObjectDefineProperty,
   Symbol,
 } = primordials;
 const { getCallerLocation } = internalBinding('util');
@@ -209,7 +211,7 @@ class Test extends AsyncResource {
     super('Test');
 
     let { fn, name, parent, skip } = options;
-    const { concurrency, loc, only, timeout, todo, signal } = options;
+    const { concurrency, loc, only, timeout, todo, signal, coverage } = options;
 
     if (typeof fn !== 'function') {
       fn = noop;
@@ -239,6 +241,7 @@ class Test extends AsyncResource {
         beforeEach: [],
         afterEach: [],
       };
+      this.coverage = coverage;
     } else {
       const nesting = parent.parent === null ? parent.nesting :
         parent.nesting + 1;
@@ -258,6 +261,7 @@ class Test extends AsyncResource {
         beforeEach: ArrayPrototypeSlice(parent.hooks.beforeEach),
         afterEach: ArrayPrototypeSlice(parent.hooks.afterEach),
       };
+      this.coverage = parent.coverage;
     }
 
     switch (typeof concurrency) {
@@ -754,6 +758,11 @@ class Test extends AsyncResource {
 
       if (coverage) {
         reporter.coverage(nesting, loc, coverage);
+        const failedCoverage = ArrayPrototypeFilter(
+          ObjectKeys(coverage.minimumCoverage), (key) => !coverage.minimumCoverage[key].status);
+        if (failedCoverage.length > 0) {
+          process.exitCode = 1;
+        }
       }
 
       reporter.end();

lib/internal/test_runner/utils.js

@@ -3,17 +3,19 @@ const {
   ArrayPrototypeJoin,
   ArrayPrototypeMap,
   ArrayPrototypeFlatMap,
+  ArrayPrototypeForEach,
   ArrayPrototypePush,
   ArrayPrototypeReduce,
   ObjectGetOwnPropertyDescriptor,
+  ObjectKeys,
   MathFloor,
   MathMax,
   MathMin,
   NumberPrototypeToFixed,
-  SafePromiseAllReturnArrayLike,
   RegExp,
   RegExpPrototypeExec,
   SafeMap,
+  SafePromiseAllReturnArrayLike,
   StringPrototypePadStart,
   StringPrototypePadEnd,
   StringPrototypeRepeat,
@@ -316,6 +318,7 @@ const kSeparator = ' | ';
 
 function getCoverageReport(pad, summary, symbol, color, table) {
   const prefix = `${pad}${symbol}`;
+  color ||= white;
   let report = `${color}${prefix}start of coverage report\n`;
 
   let filePadLength;
@@ -405,6 +408,13 @@ function getCoverageReport(pad, summary, symbol, color, table) {
             `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumnsKeys, (columnKey, j) => getCell(NumberPrototypeToFixed(summary.totals[columnKey], 2), columnPadLengths[j], StringPrototypePadStart, false, summary.totals[columnKey])), kSeparator)} |\n`;
   if (table) report += addTableLine(prefix, tableWidth);
 
+  ArrayPrototypeForEach(ObjectKeys(summary.minimumCoverage), (key) => {
+    const { actual, expected, status } = summary.minimumCoverage[key]
+    if (!status) {
+      report += `${red}${pad}${prefix}minimum coverage failed for ${key}. expected: ${expected} | actual: ${actual}\n${color}`;
+    }
+  });
+
   report += `${prefix}end of coverage report\n`;
   if (color) {
     report += white;