build: add --node-snapshot-main configure option

This adds a --build-snapshot runtime option which is currently only
supported by the node_mksnapshot binary, and a --node-snapshot-main
configure option that makes use it to run a custom script when
building the embedded snapshot. The idea is to have this experimental
feature in core as a configure-time feature for now, and investigate
the renaming V8 bugs before we make it available to more users via
making it a runtime option.

PR-URL: https://github.com/nodejs/node/pull/42466
Reviewed-By: Darshan Sen <raisinten@gmail.com>
Reviewed-By: Mohammed Keyvanzadeh <mohammadkeyvanzade94@gmail.com>
Reviewed-By: Khaidi Chu <i@2333.moe>
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>

configure.py

@@ -788,6 +788,13 @@ parser.add_argument('--node-builtin-modules-path',
     default=False,
     help='node will load builtin modules from disk instead of from binary')
 
+parser.add_argument('--node-snapshot-main',
+    action='store',
+    dest='node_snapshot_main',
+    default=None,
+    help='Run a file when building the embedded snapshot. Currently ' +
+         'experimental.')
+
 # Create compile_commands.json in out/Debug and out/Release.
 parser.add_argument('-C',
     action='store_true',
@@ -1216,6 +1223,18 @@ def configure_node(o):
 
   o['variables']['want_separate_host_toolset'] = int(cross_compiling)
 
+  if options.node_snapshot_main is not None:
+    if options.shared:
+      # This should be possible to fix, but we will need to refactor the
+      # libnode target to avoid building it twice.
+      error('--node-snapshot-main is incompatible with --shared')
+    if options.without_node_snapshot:
+      error('--node-snapshot-main is incompatible with ' +
+            '--without-node-snapshot')
+    if cross_compiling:
+      error('--node-snapshot-main is incompatible with cross compilation')
+    o['variables']['node_snapshot_main'] = options.node_snapshot_main
+
   if options.without_node_snapshot or options.node_builtin_modules_path:
     o['variables']['node_use_node_snapshot'] = 'false'
   else:

lib/internal/bootstrap/pre_execution.js

@@ -27,7 +27,8 @@ const { Buffer } = require('buffer');
 const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes;
 const assert = require('internal/assert');
 
-function prepareMainThreadExecution(expandArgv1 = false) {
+function prepareMainThreadExecution(expandArgv1 = false,
+                                    initialzeModules = true) {
   refreshRuntimeOptions();
 
   // TODO(joyeecheung): this is also necessary for workers when they deserialize
@@ -81,9 +82,13 @@ function prepareMainThreadExecution(expandArgv1 = false) {
   initializeSourceMapsHandlers();
   initializeDeprecations();
   initializeWASI();
+
+  if (!initialzeModules) {
+    return;
+  }
+
   initializeCJSLoader();
   initializeESMLoader();
-
   const CJSLoader = require('internal/modules/cjs/loader');
   assert(!CJSLoader.hasLoadedAnyUserCJSModule);
   loadPreloadModules();
@@ -102,7 +107,8 @@ function patchProcessObject(expandArgv1) {
 
   ObjectDefineProperty(process, 'argv0', {
     enumerable: true,
-    configurable: false,
+    // Only set it to true during snapshot building.
+    configurable: getOptionValue('--build-snapshot'),
     value: process.argv[0]
   });
   process.argv[0] = process.execPath;

lib/internal/main/mksnapshot.js

@@ -0,0 +1,142 @@
+'use strict';
+
+const {
+  Error,
+  SafeSet,
+  SafeArrayIterator
+} = primordials;
+
+const binding = internalBinding('mksnapshot');
+const { NativeModule } = require('internal/bootstrap/loaders');
+const {
+  compileSnapshotMain,
+} = binding;
+
+const {
+  getOptionValue
+} = require('internal/options');
+
+const {
+  readFileSync
+} = require('fs');
+
+const supportedModules = new SafeSet(new SafeArrayIterator([
+  // '_http_agent',
+  // '_http_client',
+  // '_http_common',
+  // '_http_incoming',
+  // '_http_outgoing',
+  // '_http_server',
+  '_stream_duplex',
+  '_stream_passthrough',
+  '_stream_readable',
+  '_stream_transform',
+  '_stream_wrap',
+  '_stream_writable',
+  // '_tls_common',
+  // '_tls_wrap',
+  'assert',
+  'assert/strict',
+  // 'async_hooks',
+  'buffer',
+  // 'child_process',
+  // 'cluster',
+  'console',
+  'constants',
+  'crypto',
+  // 'dgram',
+  // 'diagnostics_channel',
+  // 'dns',
+  // 'dns/promises',
+  // 'domain',
+  'events',
+  'fs',
+  'fs/promises',
+  // 'http',
+  // 'http2',
+  // 'https',
+  // 'inspector',
+  // 'module',
+  // 'net',
+  'os',
+  'path',
+  'path/posix',
+  'path/win32',
+  // 'perf_hooks',
+  'process',
+  'punycode',
+  'querystring',
+  // 'readline',
+  // 'repl',
+  'stream',
+  'stream/promises',
+  'string_decoder',
+  'sys',
+  'timers',
+  'timers/promises',
+  // 'tls',
+  // 'trace_events',
+  // 'tty',
+  'url',
+  'util',
+  'util/types',
+  'v8',
+  // 'vm',
+  // 'worker_threads',
+  // 'zlib',
+]));
+
+const warnedModules = new SafeSet();
+function supportedInUserSnapshot(id) {
+  return supportedModules.has(id);
+}
+
+function requireForUserSnapshot(id) {
+  if (!NativeModule.canBeRequiredByUsers(id)) {
+    // eslint-disable-next-line no-restricted-syntax
+    const err = new Error(
+      `Cannot find module '${id}'. `
+    );
+    err.code = 'MODULE_NOT_FOUND';
+    throw err;
+  }
+  if (!supportedInUserSnapshot(id)) {
+    if (!warnedModules.has(id)) {
+      process.emitWarning(
+        `built-in module ${id} is not yet supported in user snapshots`);
+      warnedModules.add(id);
+    }
+  }
+
+  return require(id);
+}
+
+function main() {
+  const {
+    prepareMainThreadExecution
+  } = require('internal/bootstrap/pre_execution');
+
+  prepareMainThreadExecution(true, false);
+  process.once('beforeExit', function runCleanups() {
+    for (const cleanup of binding.cleanups) {
+      cleanup();
+    }
+  });
+
+  const file = process.argv[1];
+  const path = require('path');
+  const filename = path.resolve(file);
+  const dirname = path.dirname(filename);
+  const source = readFileSync(file, 'utf-8');
+  const snapshotMainFunction = compileSnapshotMain(filename, source);
+
+  if (getOptionValue('--inspect-brk')) {
+    internalBinding('inspector').callAndPauseOnStart(
+      snapshotMainFunction, undefined,
+      requireForUserSnapshot, filename, dirname);
+  } else {
+    snapshotMainFunction(requireForUserSnapshot, filename, dirname);
+  }
+}
+
+main();

node.gyp

@@ -7,6 +7,7 @@
     'node_use_dtrace%': 'false',
     'node_use_etw%': 'false',
     'node_no_browser_globals%': 'false',
+    'node_snapshot_main%': '',
     'node_use_node_snapshot%': 'false',
     'node_use_v8_platform%': 'true',
     'node_use_bundled_v8%': 'true',
@@ -315,6 +316,28 @@
           'dependencies': [
             'node_mksnapshot',
           ],
+          'conditions': [
+            ['node_snapshot_main!=""', {
+              'actions': [
+                {
+                  'action_name': 'node_mksnapshot',
+                  'process_outputs_as_sources': 1,
+                  'inputs': [
+                    '<(node_mksnapshot_exec)',
+                    '<(node_snapshot_main)',
+                  ],
+                  'outputs': [
+                    '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc',
+                  ],
+                  'action': [
+                    '<(node_mksnapshot_exec)',
+                    '--build-snapshot',
+                    '<(node_snapshot_main)',
+                    '<@(_outputs)',
+                  ],
+                },
+              ],
+            }, {
               'actions': [
                 {
                   'action_name': 'node_mksnapshot',
@@ -331,6 +354,8 @@
                   ],
                 },
               ],
+            }],
+          ],
           }, {
           'sources': [
             'src/node_snapshot_stub.cc'

src/node.cc

@@ -495,6 +495,10 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
     return StartExecution(env, "internal/main/inspect");
   }
 
+  if (per_process::cli_options->build_snapshot) {
+    return StartExecution(env, "internal/main/mksnapshot");
+  }
+
   if (per_process::cli_options->print_help) {
     return StartExecution(env, "internal/main/print_help");
   }
@@ -1153,6 +1157,12 @@ int Start(int argc, char** argv) {
     return result.exit_code;
   }
 
+  if (per_process::cli_options->build_snapshot) {
+    fprintf(stderr,
+            "--build-snapshot is not yet supported in the node binary\n");
+    return 1;
+  }
+
   {
     bool use_node_snapshot =
         per_process::cli_options->per_isolate->node_snapshot;

src/node_binding.cc

@@ -59,6 +59,7 @@
   V(js_udp_wrap)                                                               \
   V(messaging)                                                                 \
   V(module_wrap)                                                               \
+  V(mksnapshot)                                                                \
   V(native_module)                                                             \
   V(options)                                                                   \
   V(os)                                                                        \

src/node_external_reference.h

@@ -66,6 +66,7 @@ class ExternalReferenceRegistry {
   V(handle_wrap)                                                               \
   V(heap_utils)                                                                \
   V(messaging)                                                                 \
+  V(mksnapshot)                                                                \
   V(native_module)                                                             \
   V(options)                                                                   \
   V(os)                                                                        \

src/node_options.cc

@@ -725,6 +725,11 @@ PerProcessOptionsParser::PerProcessOptionsParser(
             "disable Object.prototype.__proto__",
             &PerProcessOptions::disable_proto,
             kAllowedInEnvironment);
+  AddOption("--build-snapshot",
+            "Generate a snapshot blob when the process exits."
+            "Currently only supported in the node_mksnapshot binary.",
+            &PerProcessOptions::build_snapshot,
+            kDisallowedInEnvironment);
 
   // 12.x renamed this inadvertently, so alias it for consistency within the
   // release line, while using the original name for consistency with older

src/node_options.h

@@ -229,6 +229,7 @@ class PerProcessOptions : public Options {
   bool zero_fill_all_buffers = false;
   bool debug_arraybuffer_allocations = false;
   std::string disable_proto;
+  bool build_snapshot;
 
   std::vector<std::string> security_reverts;
   bool print_bash_completion = false;

src/node_snapshotable.cc

@@ -18,12 +18,18 @@
 namespace node {
 
 using v8::Context;
+using v8::Function;
+using v8::FunctionCallbackInfo;
 using v8::HandleScope;
 using v8::Isolate;
 using v8::Local;
+using v8::MaybeLocal;
 using v8::Object;
+using v8::ScriptCompiler;
+using v8::ScriptOrigin;
 using v8::SnapshotCreator;
 using v8::StartupData;
+using v8::String;
 using v8::TryCatch;
 using v8::Value;
 
@@ -130,16 +136,36 @@ void SnapshotBuilder::Generate(SnapshotData* out,
                             nullptr,
                             node::EnvironmentFlags::kDefaultFlags,
                             {});
+      // TODO(joyeecheung): run env->InitializeInspector({}) here.
       // Run scripts in lib/internal/bootstrap/
       {
         TryCatch bootstrapCatch(isolate);
-        v8::MaybeLocal<Value> result = env->RunBootstrapping();
+        MaybeLocal<Value> result = env->RunBootstrapping();
         if (bootstrapCatch.HasCaught()) {
           PrintCaughtException(isolate, context, bootstrapCatch);
         }
         result.ToLocalChecked();
       }
 
+      // If --build-snapshot is true, lib/internal/main/mksnapshot.js would be
+      // loaded via LoadEnvironment() to execute process.argv[1] as the entry
+      // point (we currently only support this kind of entry point, but we
+      // could also explore snapshotting other kinds of execution modes
+      // in the future).
+      if (per_process::cli_options->build_snapshot) {
+        TryCatch bootstrapCatch(isolate);
+        // TODO(joyeecheung): we could use the result for something special,
+        // like setting up initializers that should be invoked at snapshot
+        // dehydration.
+        MaybeLocal<Value> result =
+            LoadEnvironment(env, StartExecutionCallback{});
+        if (bootstrapCatch.HasCaught()) {
+          PrintCaughtException(isolate, context, bootstrapCatch);
+        }
+        result.ToLocalChecked();
+        // TODO(joyeecheung): run SpinEventLoop here.
+      }
+
       if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) {
         env->PrintAllBaseObjects();
         printf("Environment = %p\n", env);
@@ -309,4 +335,57 @@ void SerializeBindingData(Environment* env,
   });
 }
 
+namespace mksnapshot {
+
+static void CompileSnapshotMain(const FunctionCallbackInfo<Value>& args) {
+  CHECK(args[0]->IsString());
+  Local<String> filename = args[0].As<String>();
+  Local<String> source = args[1].As<String>();
+  Isolate* isolate = args.GetIsolate();
+  Local<Context> context = isolate->GetCurrentContext();
+  ScriptOrigin origin(isolate, filename, 0, 0, true);
+  // TODO(joyeecheung): do we need all of these? Maybe we would want a less
+  // internal version of them.
+  std::vector<Local<String>> parameters = {
+      FIXED_ONE_BYTE_STRING(isolate, "require"),
+      FIXED_ONE_BYTE_STRING(isolate, "__filename"),
+      FIXED_ONE_BYTE_STRING(isolate, "__dirname"),
+  };
+  ScriptCompiler::Source script_source(source, origin);
+  Local<Function> fn;
+  if (ScriptCompiler::CompileFunctionInContext(context,
+                                               &script_source,
+                                               parameters.size(),
+                                               parameters.data(),
+                                               0,
+                                               nullptr,
+                                               ScriptCompiler::kEagerCompile)
+          .ToLocal(&fn)) {
+    args.GetReturnValue().Set(fn);
+  }
+}
+
+static void Initialize(Local<Object> target,
+                       Local<Value> unused,
+                       Local<Context> context,
+                       void* priv) {
+  Environment* env = Environment::GetCurrent(context);
+  Isolate* isolate = context->GetIsolate();
+  env->SetMethod(target, "compileSnapshotMain", CompileSnapshotMain);
+  target
+      ->Set(context,
+            FIXED_ONE_BYTE_STRING(isolate, "cleanups"),
+            v8::Array::New(isolate))
+      .Check();
+}
+
+static void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
+  registry->Register(CompileSnapshotMain);
+  registry->Register(MarkBootstrapComplete);
+}
+}  // namespace mksnapshot
 }  // namespace node
+
+NODE_MODULE_CONTEXT_AWARE_INTERNAL(mksnapshot, node::mksnapshot::Initialize)
+NODE_MODULE_EXTERNAL_REFERENCE(mksnapshot,
+                               node::mksnapshot::RegisterExternalReferences)

src/node_snapshotable.h

@@ -127,6 +127,8 @@ class SnapshotBuilder {
  public:
   static std::string Generate(const std::vector<std::string> args,
                               const std::vector<std::string> exec_args);
+
+  // Generate the snapshot into out.
   static void Generate(SnapshotData* out,
                        const std::vector<std::string> args,
                        const std::vector<std::string> exec_args);

tools/snapshot/node_mksnapshot.cc

@@ -11,49 +11,87 @@
 #include "util-inl.h"
 #include "v8.h"
 
+int BuildSnapshot(int argc, char* argv[]);
+
 #ifdef _WIN32
 #include <windows.h>
 
-int wmain(int argc, wchar_t* argv[]) {
+int wmain(int argc, wchar_t* wargv[]) {
+  // Windows needs conversion from wchar_t to char.
+
+  // Convert argv to UTF8.
+  char** argv = new char*[argc + 1];
+  for (int i = 0; i < argc; i++) {
+    // Compute the size of the required buffer
+    DWORD size = WideCharToMultiByte(
+        CP_UTF8, 0, wargv[i], -1, nullptr, 0, nullptr, nullptr);
+    if (size == 0) {
+      // This should never happen.
+      fprintf(stderr, "Could not convert arguments to utf8.");
+      exit(1);
+    }
+    // Do the actual conversion
+    argv[i] = new char[size];
+    DWORD result = WideCharToMultiByte(
+        CP_UTF8, 0, wargv[i], -1, argv[i], size, nullptr, nullptr);
+    if (result == 0) {
+      // This should never happen.
+      fprintf(stderr, "Could not convert arguments to utf8.");
+      exit(1);
+    }
+  }
+  argv[argc] = nullptr;
 #else   // UNIX
 int main(int argc, char* argv[]) {
   argv = uv_setup_args(argc, argv);
+
+  // Disable stdio buffering, it interacts poorly with printf()
+  // calls elsewhere in the program (e.g., any logging from V8.)
+  setvbuf(stdout, nullptr, _IONBF, 0);
+  setvbuf(stderr, nullptr, _IONBF, 0);
 #endif  // _WIN32
 
   v8::V8::SetFlagsFromString("--random_seed=42");
+  v8::V8::SetFlagsFromString("--harmony-import-assertions");
+  return BuildSnapshot(argc, argv);
+}
 
+int BuildSnapshot(int argc, char* argv[]) {
   if (argc < 2) {
     std::cerr << "Usage: " << argv[0] << " <path/to/output.cc>\n";
+    std::cerr << "       " << argv[0] << " --build-snapshot "
+              << "<path/to/script.js> <path/to/output.cc>\n";
     return 1;
   }
 
-  std::ofstream out;
-  out.open(argv[1], std::ios::out | std::ios::binary);
-  if (!out.is_open()) {
-    std::cerr << "Cannot open " << argv[1] << "\n";
-    return 1;
-  }
-
-// Windows needs conversion from wchar_t to char. See node_main.cc
-#ifdef _WIN32
-  int node_argc = 1;
-  char argv0[] = "node";
-  char* node_argv[] = {argv0, nullptr};
-  node::InitializationResult result =
-      node::InitializeOncePerProcess(node_argc, node_argv);
-#else
   node::InitializationResult result =
       node::InitializeOncePerProcess(argc, argv);
-#endif
 
   CHECK(!result.early_return);
   CHECK_EQ(result.exit_code, 0);
 
+  std::string out_path;
+  if (node::per_process::cli_options->build_snapshot) {
+    out_path = result.args[2];
+  } else {
+    out_path = result.args[1];
+  }
+
+  std::ofstream out(out_path, std::ios::out | std::ios::binary);
+  if (!out) {
+    std::cerr << "Cannot open " << out_path << "\n";
+    return 1;
+  }
+
   {
     std::string snapshot =
         node::SnapshotBuilder::Generate(result.args, result.exec_args);
     out << snapshot;
-    out.close();
+
+    if (!out) {
+      std::cerr << "Failed to write " << out_path << "\n";
+      return 1;
+    }
   }
 
   node::TearDownOncePerProcess();