1berry/nodejs
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

build: build addon tests in parallel

Browse Source 
Use a JS script to build addons rather than a shell command
embedded in the Makefile, because parallelizing is hard in sh
and easy in JS.

PR-URL: https://github.com/nodejs/node/pull/21155
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
This commit is contained in:
Anna Henningsen 2018-06-06 00:18:55 +02:00
parent fa2d0a117e
commit cea10baa22
No known key found for this signature in database
GPG Key ID: 9C63F3A6CD2AD8F9
2 changed files with 67 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
Makefile
View File

@ -315,25 +315,14 @@ ADDONS_BINDING_SOURCES := \
# Depends on node-gyp package.json so that build-addons is (re)executed when
# node-gyp is updated as part of an npm update.
test/addons/.buildstamp: config.gypi \
deps/npm/node_modules/node-gyp/package.json \
deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \
$(ADDONS_BINDING_GYPS) $(ADDONS_BINDING_SOURCES) \
deps/uv/include/*.h deps/v8/include/*.h \
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \
test/addons/.docbuildstamp
# Cannot use $(wildcard test/addons/*/) here, it's evaluated before
# embedded addons have been generated from the documentation.
# Ignore folders without binding.gyp
# (https://github.com/nodejs/node/issues/14843)
@for dirname in test/addons/*/; do \
if [ ! -f "$$PWD/$${dirname}binding.gyp" ]; then \
continue; fi ; \
printf "\nBuilding addon $$PWD/$$dirname\n" ; \
env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \
--loglevel=$(LOGLEVEL) rebuild \
--python="$(PYTHON)" \
--directory="$$PWD/$$dirname" \
--nodedir="$$PWD" || exit 1 ; \
done
env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \
npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \
"$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" "$$PWD/test/addons"
touch $@
.PHONY: build-addons
@ -355,25 +344,15 @@ ADDONS_NAPI_BINDING_SOURCES := \
# Implicitly depends on $(NODE_EXE), see the build-addons-napi rule for rationale.
test/addons-napi/.buildstamp: config.gypi \
deps/npm/node_modules/node-gyp/package.json \
deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \
$(ADDONS_NAPI_BINDING_GYPS) $(ADDONS_NAPI_BINDING_SOURCES) \
deps/uv/include/*.h deps/v8/include/*.h \
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \
src/node_api.h src/node_api_types.h
# Cannot use $(wildcard test/addons-napi/*/) here, it's evaluated before
# embedded addons have been generated from the documentation.
# Ignore folders without binding.gyp
# (https://github.com/nodejs/node/issues/14843)
@for dirname in test/addons-napi/*/; do \
if [ ! -f "$$PWD/$${dirname}binding.gyp" ]; then \
continue; fi ; \
printf "\nBuilding addon $$PWD/$$dirname\n" ; \
env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \
--loglevel=$(LOGLEVEL) rebuild \
--python="$(PYTHON)" \
--directory="$$PWD/$$dirname" \
--nodedir="$$PWD" || exit 1 ; \
done
env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \
npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \
"$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \
"$$PWD/test/addons-napi"
touch $@
.PHONY: build-addons-napi

58
tools/build-addons.js Normal file
View File

@ -0,0 +1,58 @@
'use strict';
// Usage: e.g. node build-addons.js <path to node-gyp> <directory>
const child_process = require('child_process');
const path = require('path');
const fs = require('fs').promises;
const util = require('util');
const execFile = util.promisify(child_process.execFile);
const parallelization = +process.env.JOBS || require('os').cpus().length;
const nodeGyp = process.argv[2];
async function runner(directoryQueue) {
if (directoryQueue.length === 0)
return;
const dir = directoryQueue.shift();
const next = () => runner(directoryQueue);
try {
// Only run for directories that have a `binding.gyp`.
// (https://github.com/nodejs/node/issues/14843)
await fs.stat(path.join(dir, 'binding.gyp'));
} catch (err) {
if (err.code === 'ENOENT' || err.code === 'ENOTDIR')
return next();
throw err;
}
console.log(`Building addon in ${dir}`);
const { stdout, stderr } =
await execFile(process.execPath, [nodeGyp, 'rebuild', `--directory=${dir}`],
{
stdio: 'inherit',
env: { ...process.env, MAKEFLAGS: '-j1' }
});
// We buffer the output and print it out once the process is done in order
// to avoid interleaved output from multiple builds running at once.
process.stdout.write(stdout);
process.stderr.write(stderr);
return next();
}
async function main(directory) {
const directoryQueue = (await fs.readdir(directory))
.map((subdir) => path.join(directory, subdir));
const runners = [];
for (let i = 0; i < parallelization; ++i)
runners.push(runner(directoryQueue));
return Promise.all(runners);
}
main(process.argv[3]).catch((err) => setImmediate(() => { throw err; }));