test: switch to native v8 coverage

PR-URL: https://github.com/nodejs/node/pull/25157
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Sakthipriyan Vairamani <thechargingvolcano@gmail.com>
Reviewed-By: Refael Ackermann <refack@gmail.com>

BUILDING.md

@@ -290,9 +290,15 @@ $ CI_JS_SUITES=child-process CI_NATIVE_SUITES= make coverage
 The above command executes tests for the `child-process` subsystem and
 outputs the resulting coverage report.
 
-The `make coverage` command downloads some tools to the project root directory
+Alternatively, for the JavaScript test suite, you can use the `CI_JS_SUITES`
-and overwrites the `lib/` directory. To clean up after generating the coverage
+variable to run tests in isolation, outputting reports:
-reports:
+
+```text
+$ CI_JS_SUITES=fs CI_NATIVE_SUITES= make coverage-run-js
+```
+
+The `make coverage` command downloads some tools to the project root directory.
+To clean up after generating the coverage reports:
 
 ```console
 $ make coverage-clean

Makefile

@@ -11,6 +11,7 @@ STAGINGSERVER ?= node-www
 LOGLEVEL ?= silent
 OSTYPE := $(shell uname -s | tr '[A-Z]' '[a-z]')
 COVTESTS ?= test-cov
+COV_SKIP_TESTS ?= core_line_numbers.js,testFinalizer.js,test_function/test.js
 GTEST_FILTER ?= "*"
 GNUMAKEFLAGS += --no-print-directory
 GCOV ?= gcov
@@ -181,7 +182,6 @@ coverage-clean:
 	$(RM) -r node_modules
 	$(RM) -r gcovr build
 	$(RM) -r out/$(BUILDTYPE)/.coverage
-	$(RM) -r .cov_tmp
 	$(RM) out/$(BUILDTYPE)/obj.target/node/gen/*.gcda
 	$(RM) out/$(BUILDTYPE)/obj.target/node/src/*.gcda
 	$(RM) out/$(BUILDTYPE)/obj.target/node/src/tracing/*.gcda
@@ -201,9 +201,7 @@ coverage: coverage-test ## Run the tests and generate a coverage report.
 
 .PHONY: coverage-build
 coverage-build: all
-	mkdir -p node_modules
+	-$(MAKE) coverage-build-js
-	if [ ! -d node_modules/nyc ]; then \
-		$(NODE) ./deps/npm install nyc@13 --no-save --no-package-lock; fi
 	if [ ! -d gcovr ]; then git clone -b 3.4 --depth=1 \
 		--single-branch https://github.com/gcovr/gcovr.git; fi
 	if [ ! -d build ]; then git clone --depth=1 \
@@ -211,38 +209,29 @@ coverage-build: all
 	if [ ! -f gcovr/scripts/gcovr.orig ]; then \
 		(cd gcovr && patch -N -p1 < \
 		"$(CURDIR)/build/jenkins/scripts/coverage/gcovr-patches-3.4.diff"); fi
-	if [ -d lib_ ]; then $(RM) -r lib; mv lib_ lib; fi
-	mv lib lib_
-	NODE_DEBUG=nyc $(NODE) ./node_modules/.bin/nyc instrument --extension .js \
-		--extension .mjs --exit-on-error lib_/ lib/
 	$(MAKE)
 
+.PHONY: coverage-build-js
+coverage-build-js:
+	mkdir -p node_modules
+	if [ ! -d node_modules/c8 ]; then \
+		$(NODE) ./deps/npm install c8@next --no-save --no-package-lock;\
+	fi
+
 .PHONY: coverage-test
 coverage-test: coverage-build
-	$(RM) -r out/$(BUILDTYPE)/.coverage
-	$(RM) -r .cov_tmp
 	$(RM) out/$(BUILDTYPE)/obj.target/node/gen/*.gcda
 	$(RM) out/$(BUILDTYPE)/obj.target/node/src/*.gcda
 	$(RM) out/$(BUILDTYPE)/obj.target/node/src/tracing/*.gcda
 	$(RM) out/$(BUILDTYPE)/obj.target/node_lib/gen/*.gcda
 	$(RM) out/$(BUILDTYPE)/obj.target/node_lib/src/*.gcda
 	$(RM) out/$(BUILDTYPE)/obj.target/node_lib/src/tracing/*.gcda
-	-$(MAKE) $(COVTESTS)
+	-NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage $(MAKE) $(COVTESTS)
-	mv lib lib__
+	$(MAKE) coverage-report-js
-	mv lib_ lib
-	mkdir -p coverage .cov_tmp
-	$(NODE) ./node_modules/.bin/nyc merge 'out/Release/.coverage' \
-		.cov_tmp/libcov.json
-	(cd lib && .$(NODE) ../node_modules/.bin/nyc report \
-		--temp-dir "$(CURDIR)/.cov_tmp" \
-		--report-dir "$(CURDIR)/coverage" \
-		--reporter html)
 	-(cd out && "../gcovr/scripts/gcovr" --gcov-exclude='.*deps' \
 		--gcov-exclude='.*usr' -v -r Release/obj.target \
 		--html --html-detail -o ../coverage/cxxcoverage.html \
 		--gcov-executable="$(GCOV)")
-	mv lib lib_
-	mv lib__ lib
 	@echo -n "Javascript coverage %: "
 	@grep -B1 Lines coverage/index.html | head -n1 \
 		| sed 's/<[^>]*>//g'| sed 's/ //g'
@@ -250,6 +239,13 @@ coverage-test: coverage-build
 	@grep -A3 Lines coverage/cxxcoverage.html | grep style  \
 		| sed 's/<[^>]*>//g'| sed 's/ //g'
 
+.PHONY: coverage-report-js
+coverage-report-js:
+	$(NODE) ./node_modules/.bin/c8 report --reporter=html \
+		--temp-directory=out/$(BUILDTYPE)/.coverage --omit-relative=false \
+		--resolve=./lib --exclude="deps/" --exclude="test/" --exclude="tools/" \
+		--wrapper-length=0
+
 .PHONY: cctest
 # Runs the C++ tests using the built `cctest` executable.
 cctest: all
@@ -276,6 +272,14 @@ jstest: build-addons build-js-native-api-tests build-node-api-tests ## Runs addo
 		$(CI_JS_SUITES) \
 		$(CI_NATIVE_SUITES)
 
+.PHONY: coverage-run-js
+coverage-run-js:
+	$(RM) -r out/$(BUILDTYPE)/.coverage
+	$(MAKE) coverage-build-js
+	-NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage CI_SKIP_TESTS=$(COV_SKIP_TESTS) \
+	  $(MAKE) jstest
+	$(MAKE) coverage-report-js
+
 .PHONY: test
 # This does not run tests of third-party libraries inside deps.
 test: all ## Runs default tests, linters, and builds docs.
@@ -300,7 +304,7 @@ test-cov: all
 	$(MAKE) build-js-native-api-tests
 	$(MAKE) build-node-api-tests
 	# $(MAKE) cctest
-	CI_SKIP_TESTS=core_line_numbers.js $(MAKE) jstest
+	CI_SKIP_TESTS=$(COV_SKIP_TESTS) $(MAKE) jstest
 
 test-parallel: all
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) --mode=$(BUILDTYPE_LOWER) parallel

lib/internal/bootstrap/node.js

@@ -318,26 +318,10 @@ function startup() {
     }
   });
 
-  // Set up coverage exit hooks.
-  let originalReallyExit = process.reallyExit;
-  // Core coverage generation using nyc instrumented lib/ files.
-  // See `make coverage-build`. This does not affect user land.
-  // TODO(joyeecheung): this and `with_instrumentation.js` can be
-  // removed in favor of NODE_V8_COVERAGE once we switch to that
-  // in https://coverage.nodejs.org/
-  if (global.__coverage__) {
-    const {
-      writeCoverage
-    } = NativeModule.require('internal/coverage-gen/with_instrumentation');
-    process.on('exit', writeCoverage);
-    originalReallyExit = process.reallyExit = (code) => {
-      writeCoverage();
-      originalReallyExit(code);
-    };
-  }
   // User-facing NODE_V8_COVERAGE environment variable that writes
   // ScriptCoverage to a specified file.
   if (process.env.NODE_V8_COVERAGE) {
+    const originalReallyExit = process.reallyExit;
     const cwd = NativeModule.require('internal/process/execution').tryGetCwd();
     const { resolve } = NativeModule.require('path');
     // Resolve the coverage directory to an absolute path, and

lib/internal/coverage-gen/with_instrumentation.js

@@ -1,36 +0,0 @@
-'use strict';
-
-// This file contains hooks for nyc instrumented lib/ files to collect
-// JS coverage for core.
-// See `make coverage-build`.
-function writeCoverage() {
-  if (!global.__coverage__) {
-    return;
-  }
-
-  const path = require('path');
-  const { mkdirSync, writeFileSync } = require('fs');
-
-  const dirname = path.join(path.dirname(process.execPath), '.coverage');
-  const filename = `coverage-${process.pid}-${Date.now()}.json`;
-  try {
-    mkdirSync(dirname);
-  } catch (err) {
-    if (err.code !== 'EEXIST') {
-      console.error(err);
-      return;
-    }
-  }
-
-  const target = path.join(dirname, filename);
-  const coverageInfo = JSON.stringify(global.__coverage__);
-  try {
-    writeFileSync(target, coverageInfo);
-  } catch (err) {
-    console.error(err);
-  }
-}
-
-module.exports = {
-  writeCoverage
-};

node.gyp

@@ -99,7 +99,6 @@
       'lib/internal/console/global.js',
       'lib/internal/console/inspector.js',
       'lib/internal/coverage-gen/with_profiler.js',
-      'lib/internal/coverage-gen/with_instrumentation.js',
       'lib/internal/crypto/certificate.js',
       'lib/internal/crypto/cipher.js',
       'lib/internal/crypto/diffiehellman.js',

test/common/index.js

@@ -238,9 +238,6 @@ function platformTimeout(ms) {
   if (process.features.debug)
     ms = multipliers.two * ms;
 
-  if (global.__coverage__)
-    ms = multipliers.four * ms;
-
   if (isAIX)
     return multipliers.two * ms; // default localhost speed is slower on AIX
 
@@ -299,12 +296,8 @@ function leakedGlobals() {
     }
   }
 
-  if (global.__coverage__) {
-    return leaked.filter((varname) => !/^(?:cov_|__cov)/.test(varname));
-  } else {
   return leaked;
 }
-}
 
 process.on('exit', function() {
   const leaked = leakedGlobals();

test/parallel/test-bootstrap-modules.js

@@ -9,7 +9,8 @@ const common = require('../common');
 const assert = require('assert');
 
 const isMainThread = common.isMainThread;
-const kMaxModuleCount = isMainThread ? 63 : 85;
+const kCoverageModuleCount = process.env.NODE_V8_COVERAGE ? 1 : 0;
+const kMaxModuleCount = (isMainThread ? 63 : 85) + kCoverageModuleCount;
 
 assert(list.length <= kMaxModuleCount,
        `Total length: ${list.length}\n` + list.join('\n')