1berry/nodejs
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

build: add --node-snapshot-main configure option

Browse Source 
This adds a --build-snapshot runtime option which is currently only
supported by the node_mksnapshot binary, and a --node-snapshot-main
configure option that makes use it to run a custom script when
building the embedded snapshot. The idea is to have this experimental
feature in core as a configure-time feature for now, and investigate
the renaming V8 bugs before we make it available to more users via
making it a runtime option.

PR-URL: https://github.com/nodejs/node/pull/42466
Reviewed-By: Darshan Sen <raisinten@gmail.com>
Reviewed-By: Mohammed Keyvanzadeh <mohammadkeyvanzade94@gmail.com>
Reviewed-By: Khaidi Chu <i@2333.moe>
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
This commit is contained in:
Joyee Cheung 2022-03-24 21:37:08 +08:00
parent 8a297ba3a0
commit 37aee80643
No known key found for this signature in database
GPG Key ID: 92B78A53C8303B8D
12 changed files with 365 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
configure.py
View File

@ -788,6 +788,13 @@ parser.add_argument('--node-builtin-modules-path',
default=False,
help='node will load builtin modules from disk instead of from binary')
parser.add_argument('--node-snapshot-main',
action='store',
dest='node_snapshot_main',
default=None,
help='Run a file when building the embedded snapshot. Currently ' +
'experimental.')
# Create compile_commands.json in out/Debug and out/Release.
parser.add_argument('-C',
action='store_true',
@ -1216,6 +1223,18 @@ def configure_node(o):
o['variables']['want_separate_host_toolset'] = int(cross_compiling)
if options.node_snapshot_main is not None:
if options.shared:
# This should be possible to fix, but we will need to refactor the
# libnode target to avoid building it twice.
error('--node-snapshot-main is incompatible with --shared')
if options.without_node_snapshot:
error('--node-snapshot-main is incompatible with ' +
'--without-node-snapshot')
if cross_compiling:
error('--node-snapshot-main is incompatible with cross compilation')
o['variables']['node_snapshot_main'] = options.node_snapshot_main
if options.without_node_snapshot or options.node_builtin_modules_path:
o['variables']['node_use_node_snapshot'] = 'false'
else:

12
lib/internal/bootstrap/pre_execution.js
View File

@ -27,7 +27,8 @@ const { Buffer } = require('buffer');
const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes;
const assert = require('internal/assert');
function prepareMainThreadExecution(expandArgv1 = false) {
function prepareMainThreadExecution(expandArgv1 = false,
initialzeModules = true) {
refreshRuntimeOptions();
// TODO(joyeecheung): this is also necessary for workers when they deserialize
@ -81,9 +82,13 @@ function prepareMainThreadExecution(expandArgv1 = false) {
initializeSourceMapsHandlers();
initializeDeprecations();
initializeWASI();
if (!initialzeModules) {
return;
}
initializeCJSLoader();
initializeESMLoader();
const CJSLoader = require('internal/modules/cjs/loader');
assert(!CJSLoader.hasLoadedAnyUserCJSModule);
loadPreloadModules();
@ -102,7 +107,8 @@ function patchProcessObject(expandArgv1) {
ObjectDefineProperty(process, 'argv0', {
enumerable: true,
configurable: false,
// Only set it to true during snapshot building.
configurable: getOptionValue('--build-snapshot'),
value: process.argv[0]
});
process.argv[0] = process.execPath;

142
lib/internal/main/mksnapshot.js Normal file
View File

@ -0,0 +1,142 @@
'use strict';
const {
Error,
SafeSet,
SafeArrayIterator
} = primordials;
const binding = internalBinding('mksnapshot');
const { NativeModule } = require('internal/bootstrap/loaders');
const {
compileSnapshotMain,
} = binding;
const {
getOptionValue
} = require('internal/options');
const {
readFileSync
} = require('fs');
const supportedModules = new SafeSet(new SafeArrayIterator([
// '_http_agent',
// '_http_client',
// '_http_common',
// '_http_incoming',
// '_http_outgoing',
// '_http_server',
'_stream_duplex',
'_stream_passthrough',
'_stream_readable',
'_stream_transform',
'_stream_wrap',
'_stream_writable',
// '_tls_common',
// '_tls_wrap',
'assert',
'assert/strict',
// 'async_hooks',
'buffer',
// 'child_process',
// 'cluster',
'console',
'constants',
'crypto',
// 'dgram',
// 'diagnostics_channel',
// 'dns',
// 'dns/promises',
// 'domain',
'events',
'fs',
'fs/promises',
// 'http',
// 'http2',
// 'https',
// 'inspector',
// 'module',
// 'net',
'os',
'path',
'path/posix',
'path/win32',
// 'perf_hooks',
'process',
'punycode',
'querystring',
// 'readline',
// 'repl',
'stream',
'stream/promises',
'string_decoder',
'sys',
'timers',
'timers/promises',
// 'tls',
// 'trace_events',
// 'tty',
'url',
'util',
'util/types',
'v8',
// 'vm',
// 'worker_threads',
// 'zlib',
]));
const warnedModules = new SafeSet();
function supportedInUserSnapshot(id) {
return supportedModules.has(id);
}
function requireForUserSnapshot(id) {
if (!NativeModule.canBeRequiredByUsers(id)) {
// eslint-disable-next-line no-restricted-syntax
const err = new Error(
`Cannot find module '${id}'. `
);
err.code = 'MODULE_NOT_FOUND';
throw err;
}
if (!supportedInUserSnapshot(id)) {
if (!warnedModules.has(id)) {
process.emitWarning(
`built-in module ${id} is not yet supported in user snapshots`);
warnedModules.add(id);
}
}
return require(id);
}
function main() {
const {
prepareMainThreadExecution
} = require('internal/bootstrap/pre_execution');
prepareMainThreadExecution(true, false);
process.once('beforeExit', function runCleanups() {
for (const cleanup of binding.cleanups) {
cleanup();
}
});
const file = process.argv[1];
const path = require('path');
const filename = path.resolve(file);
const dirname = path.dirname(filename);
const source = readFileSync(file, 'utf-8');
const snapshotMainFunction = compileSnapshotMain(filename, source);
if (getOptionValue('--inspect-brk')) {
internalBinding('inspector').callAndPauseOnStart(
snapshotMainFunction, undefined,
requireForUserSnapshot, filename, dirname);
} else {
snapshotMainFunction(requireForUserSnapshot, filename, dirname);
}
}
main();

53
node.gyp
View File

@ -7,6 +7,7 @@
'node_use_dtrace%': 'false',
'node_use_etw%': 'false',
'node_no_browser_globals%': 'false',
'node_snapshot_main%': '',
'node_use_node_snapshot%': 'false',
'node_use_v8_platform%': 'true',
'node_use_bundled_v8%': 'true',
@ -315,23 +316,47 @@
'dependencies': [
'node_mksnapshot',
],
'actions': [
{
'action_name': 'node_mksnapshot',
'process_outputs_as_sources': 1,
'inputs': [
'<(node_mksnapshot_exec)',
'conditions': [
['node_snapshot_main!=""', {
'actions': [
{
'action_name': 'node_mksnapshot',
'process_outputs_as_sources': 1,
'inputs': [
'<(node_mksnapshot_exec)',
'<(node_snapshot_main)',
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc',
],
'action': [
'<(node_mksnapshot_exec)',
'--build-snapshot',
'<(node_snapshot_main)',
'<@(_outputs)',
],
},
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc',
}, {
'actions': [
{
'action_name': 'node_mksnapshot',
'process_outputs_as_sources': 1,
'inputs': [
'<(node_mksnapshot_exec)',
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc',
],
'action': [
'<@(_inputs)',
'<@(_outputs)',
],
},
],
'action': [
'<@(_inputs)',
'<@(_outputs)',
],
},
}],
],
}, {
}, {
'sources': [
'src/node_snapshot_stub.cc'
],

10
src/node.cc
View File

@ -495,6 +495,10 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
return StartExecution(env, "internal/main/inspect");
}
if (per_process::cli_options->build_snapshot) {
return StartExecution(env, "internal/main/mksnapshot");
}
if (per_process::cli_options->print_help) {
return StartExecution(env, "internal/main/print_help");
}
@ -1153,6 +1157,12 @@ int Start(int argc, char** argv) {
return result.exit_code;
}
if (per_process::cli_options->build_snapshot) {
fprintf(stderr,
"--build-snapshot is not yet supported in the node binary\n");
return 1;
}
{
bool use_node_snapshot =
per_process::cli_options->per_isolate->node_snapshot;

1
src/node_binding.cc
View File

@ -59,6 +59,7 @@
V(js_udp_wrap) \
V(messaging) \
V(module_wrap) \
V(mksnapshot) \
V(native_module) \
V(options) \
V(os) \

1
src/node_external_reference.h
View File

@ -66,6 +66,7 @@ class ExternalReferenceRegistry {
V(handle_wrap) \
V(heap_utils) \
V(messaging) \
V(mksnapshot) \
V(native_module) \
V(options) \
V(os) \

5
src/node_options.cc
View File

@ -725,6 +725,11 @@ PerProcessOptionsParser::PerProcessOptionsParser(
"disable Object.prototype.__proto__",
&PerProcessOptions::disable_proto,
kAllowedInEnvironment);
AddOption("--build-snapshot",
"Generate a snapshot blob when the process exits."
"Currently only supported in the node_mksnapshot binary.",
&PerProcessOptions::build_snapshot,
kDisallowedInEnvironment);
// 12.x renamed this inadvertently, so alias it for consistency within the
// release line, while using the original name for consistency with older

1
src/node_options.h
View File

@ -229,6 +229,7 @@ class PerProcessOptions : public Options {
bool zero_fill_all_buffers = false;
bool debug_arraybuffer_allocations = false;
std::string disable_proto;
bool build_snapshot;
std::vector<std::string> security_reverts;
bool print_bash_completion = false;

81
src/node_snapshotable.cc
View File

@ -18,12 +18,18 @@
namespace node {
using v8::Context;
using v8::Function;
using v8::FunctionCallbackInfo;
using v8::HandleScope;
using v8::Isolate;
using v8::Local;
using v8::MaybeLocal;
using v8::Object;
using v8::ScriptCompiler;
using v8::ScriptOrigin;
using v8::SnapshotCreator;
using v8::StartupData;
using v8::String;
using v8::TryCatch;
using v8::Value;
@ -130,16 +136,36 @@ void SnapshotBuilder::Generate(SnapshotData* out,
nullptr,
node::EnvironmentFlags::kDefaultFlags,
{});
// TODO(joyeecheung): run env->InitializeInspector({}) here.
// Run scripts in lib/internal/bootstrap/
{
TryCatch bootstrapCatch(isolate);
v8::MaybeLocal<Value> result = env->RunBootstrapping();
MaybeLocal<Value> result = env->RunBootstrapping();
if (bootstrapCatch.HasCaught()) {
PrintCaughtException(isolate, context, bootstrapCatch);
}
result.ToLocalChecked();
}
// If --build-snapshot is true, lib/internal/main/mksnapshot.js would be
// loaded via LoadEnvironment() to execute process.argv[1] as the entry
// point (we currently only support this kind of entry point, but we
// could also explore snapshotting other kinds of execution modes
// in the future).
if (per_process::cli_options->build_snapshot) {
TryCatch bootstrapCatch(isolate);
// TODO(joyeecheung): we could use the result for something special,
// like setting up initializers that should be invoked at snapshot
// dehydration.
MaybeLocal<Value> result =
LoadEnvironment(env, StartExecutionCallback{});
if (bootstrapCatch.HasCaught()) {
PrintCaughtException(isolate, context, bootstrapCatch);
}
result.ToLocalChecked();
// TODO(joyeecheung): run SpinEventLoop here.
}
if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) {
env->PrintAllBaseObjects();
printf("Environment = %p\n", env);
@ -309,4 +335,57 @@ void SerializeBindingData(Environment* env,
});
}
namespace mksnapshot {
static void CompileSnapshotMain(const FunctionCallbackInfo<Value>& args) {
CHECK(args[0]->IsString());
Local<String> filename = args[0].As<String>();
Local<String> source = args[1].As<String>();
Isolate* isolate = args.GetIsolate();
Local<Context> context = isolate->GetCurrentContext();
ScriptOrigin origin(isolate, filename, 0, 0, true);
// TODO(joyeecheung): do we need all of these? Maybe we would want a less
// internal version of them.
std::vector<Local<String>> parameters = {
FIXED_ONE_BYTE_STRING(isolate, "require"),
FIXED_ONE_BYTE_STRING(isolate, "__filename"),
FIXED_ONE_BYTE_STRING(isolate, "__dirname"),
};
ScriptCompiler::Source script_source(source, origin);
Local<Function> fn;
if (ScriptCompiler::CompileFunctionInContext(context,
&script_source,
parameters.size(),
parameters.data(),
0,
nullptr,
ScriptCompiler::kEagerCompile)
.ToLocal(&fn)) {
args.GetReturnValue().Set(fn);
}
}
static void Initialize(Local<Object> target,
Local<Value> unused,
Local<Context> context,
void* priv) {
Environment* env = Environment::GetCurrent(context);
Isolate* isolate = context->GetIsolate();
env->SetMethod(target, "compileSnapshotMain", CompileSnapshotMain);
target
->Set(context,
FIXED_ONE_BYTE_STRING(isolate, "cleanups"),
v8::Array::New(isolate))
.Check();
}
static void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
registry->Register(CompileSnapshotMain);
registry->Register(MarkBootstrapComplete);
}
} // namespace mksnapshot
} // namespace node
NODE_MODULE_CONTEXT_AWARE_INTERNAL(mksnapshot, node::mksnapshot::Initialize)
NODE_MODULE_EXTERNAL_REFERENCE(mksnapshot,
node::mksnapshot::RegisterExternalReferences)

2
src/node_snapshotable.h
View File

@ -127,6 +127,8 @@ class SnapshotBuilder {
public:
static std::string Generate(const std::vector<std::string> args,
const std::vector<std::string> exec_args);
// Generate the snapshot into out.
static void Generate(SnapshotData* out,
const std::vector<std::string> args,
const std::vector<std::string> exec_args);

74
tools/snapshot/node_mksnapshot.cc
View File

@ -11,49 +11,87 @@
#include "util-inl.h"
#include "v8.h"
int BuildSnapshot(int argc, char* argv[]);
#ifdef _WIN32
#include <windows.h>
int wmain(int argc, wchar_t* argv[]) {
int wmain(int argc, wchar_t* wargv[]) {
// Windows needs conversion from wchar_t to char.
// Convert argv to UTF8.
char** argv = new char*[argc + 1];
for (int i = 0; i < argc; i++) {
// Compute the size of the required buffer
DWORD size = WideCharToMultiByte(
CP_UTF8, 0, wargv[i], -1, nullptr, 0, nullptr, nullptr);
if (size == 0) {
// This should never happen.
fprintf(stderr, "Could not convert arguments to utf8.");
exit(1);
}
// Do the actual conversion
argv[i] = new char[size];
DWORD result = WideCharToMultiByte(
CP_UTF8, 0, wargv[i], -1, argv[i], size, nullptr, nullptr);
if (result == 0) {
// This should never happen.
fprintf(stderr, "Could not convert arguments to utf8.");
exit(1);
}
}
argv[argc] = nullptr;
#else // UNIX
int main(int argc, char* argv[]) {
argv = uv_setup_args(argc, argv);
// Disable stdio buffering, it interacts poorly with printf()
// calls elsewhere in the program (e.g., any logging from V8.)
setvbuf(stdout, nullptr, _IONBF, 0);
setvbuf(stderr, nullptr, _IONBF, 0);
#endif // _WIN32
v8::V8::SetFlagsFromString("--random_seed=42");
v8::V8::SetFlagsFromString("--harmony-import-assertions");
return BuildSnapshot(argc, argv);
}
int BuildSnapshot(int argc, char* argv[]) {
if (argc < 2) {
std::cerr << "Usage: " << argv[0] << " <path/to/output.cc>\n";
std::cerr << " " << argv[0] << " --build-snapshot "
<< "<path/to/script.js> <path/to/output.cc>\n";
return 1;
}
std::ofstream out;
out.open(argv[1], std::ios::out | std::ios::binary);
if (!out.is_open()) {
std::cerr << "Cannot open " << argv[1] << "\n";
return 1;
}
// Windows needs conversion from wchar_t to char. See node_main.cc
#ifdef _WIN32
int node_argc = 1;
char argv0[] = "node";
char* node_argv[] = {argv0, nullptr};
node::InitializationResult result =
node::InitializeOncePerProcess(node_argc, node_argv);
#else
node::InitializationResult result =
node::InitializeOncePerProcess(argc, argv);
#endif
CHECK(!result.early_return);
CHECK_EQ(result.exit_code, 0);
std::string out_path;
if (node::per_process::cli_options->build_snapshot) {
out_path = result.args[2];
} else {
out_path = result.args[1];
}
std::ofstream out(out_path, std::ios::out | std::ios::binary);
if (!out) {
std::cerr << "Cannot open " << out_path << "\n";
return 1;
}
{
std::string snapshot =
node::SnapshotBuilder::Generate(result.args, result.exec_args);
out << snapshot;
out.close();
if (!out) {
std::cerr << "Failed to write " << out_path << "\n";
return 1;
}
}
node::TearDownOncePerProcess();