src: port bootstrap/cache.js to C++

This allows us to query the categories of modules in C++
so we can implement the code cache generator in C++ that
does not depend on a Node.js binary.

PR-URL: https://github.com/nodejs/node/pull/27046
Refs: https://github.com/nodejs/node/issues/21563
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>

benchmark/fixtures/require-cachable.js

@@ -1,13 +1,10 @@
 'use strict';
 
-const list = require('internal/bootstrap/cache');
+const { internalBinding } = require('internal/test/binding');
 const {
-  isMainThread
+  moduleCategories: { canBeRequired }
-} = require('worker_threads');
+} = internalBinding('native_module');
 
-for (const key of list.cachableBuiltins) {
+for (const key of canBeRequired) {
-  if (!isMainThread && key === 'trace_events') {
-    continue;
-  }
   require(key);
 }

lib/internal/bootstrap/cache.js

@@ -1,85 +0,0 @@
-'use strict';
-
-// This is only exposed for internal build steps and testing purposes.
-// We create new copies of the source and the code cache
-// so the resources eventually used to compile builtin modules
-// cannot be tampered with even with --expose-internals.
-
-const { NativeModule } = require('internal/bootstrap/loaders');
-const {
-  getCodeCache, compileFunction
-} = internalBinding('native_module');
-const { hasTracing, hasInspector } = internalBinding('config');
-
-// Modules with source code compiled in js2c that
-// cannot be compiled with the code cache.
-const cannotBeRequired = [
-  'sys',  // Deprecated.
-  'internal/v8_prof_polyfill',
-  'internal/v8_prof_processor',
-
-  'internal/test/binding',
-
-  'internal/bootstrap/environment',
-  'internal/bootstrap/primordials',
-  'internal/bootstrap/loaders',
-  'internal/bootstrap/node',
-
-  'internal/per_context/setup',
-  'internal/per_context/domexception',
-];
-
-// Skip modules that cannot be required when they are not
-// built into the binary.
-if (!hasInspector) {
-  cannotBeRequired.push(
-    'inspector',
-    'internal/util/inspector',
-  );
-}
-if (!hasTracing) {
-  cannotBeRequired.push('trace_events');
-}
-if (!process.versions.openssl) {
-  cannotBeRequired.push(
-    'crypto',
-    'https',
-    'http2',
-    'tls',
-    '_tls_common',
-    '_tls_wrap',
-    'internal/crypto/certificate',
-    'internal/crypto/cipher',
-    'internal/crypto/diffiehellman',
-    'internal/crypto/hash',
-    'internal/crypto/keygen',
-    'internal/crypto/keys',
-    'internal/crypto/pbkdf2',
-    'internal/crypto/random',
-    'internal/crypto/scrypt',
-    'internal/crypto/sig',
-    'internal/crypto/util',
-    'internal/http2/core',
-    'internal/http2/compat',
-    'internal/policy/manifest',
-    'internal/process/policy',
-    'internal/streams/lazy_transform',
-  );
-}
-
-const cachableBuiltins = [];
-for (const id of NativeModule.map.keys()) {
-  if (id.startsWith('internal/deps') || id.startsWith('internal/main')) {
-    cannotBeRequired.push(id);
-  }
-  if (!cannotBeRequired.includes(id)) {
-    cachableBuiltins.push(id);
-  }
-}
-
-module.exports = {
-  cachableBuiltins,
-  getCodeCache,
-  compileFunction,
-  cannotBeRequired
-};

node.gyp

@@ -27,7 +27,6 @@
     'node_intermediate_lib_type%': 'static_library',
     'library_files': [
       'lib/internal/bootstrap/primordials.js',
-      'lib/internal/bootstrap/cache.js',
       'lib/internal/bootstrap/environment.js',
       'lib/internal/bootstrap/loaders.js',
       'lib/internal/bootstrap/node.js',

src/node_native_module.cc

@@ -36,6 +36,75 @@ using v8::String;
 using v8::Uint8Array;
 using v8::Value;
 
+void NativeModuleLoader::InitializeModuleCategories() {
+  if (module_categories_.is_initialized) {
+    DCHECK(!module_categories_.can_be_required.empty());
+    return;
+  }
+
+  std::vector<std::string> prefixes = {
+#if !HAVE_OPENSSL
+    "internal/crypto/",
+#endif  // !HAVE_OPENSSL
+
+    "internal/bootstrap/",
+    "internal/per_context/",
+    "internal/deps/",
+    "internal/main/"
+  };
+
+  module_categories_.cannot_be_required = std::set<std::string> {
+#if !HAVE_INSPECTOR
+      "inspector",
+      "internal/util/inspector",
+#endif  // !HAVE_INSPECTOR
+
+#if !NODE_USE_V8_PLATFORM || !defined(NODE_HAVE_I18N_SUPPORT)
+      "trace_events",
+#endif  // !NODE_USE_V8_PLATFORM
+
+#if !HAVE_OPENSSL
+      "crypto",
+      "https",
+      "http2",
+      "tls",
+      "_tls_common",
+      "_tls_wrap",
+      "internal/http2/core",
+      "internal/http2/compat",
+      "internal/policy/manifest",
+      "internal/process/policy",
+      "internal/streams/lazy_transform",
+#endif  // !HAVE_OPENSSL
+
+      "sys",  // Deprecated.
+      "internal/test/binding",
+      "internal/v8_prof_polyfill",
+      "internal/v8_prof_processor",
+  };
+
+  for (auto const& x : source_) {
+    const std::string& id = x.first;
+    for (auto const& prefix : prefixes) {
+      if (prefix.length() > id.length()) {
+        continue;
+      }
+      if (id.find(prefix) == 0) {
+        module_categories_.cannot_be_required.emplace(id);
+      }
+    }
+  }
+
+  for (auto const& x : source_) {
+    const std::string& id = x.first;
+    if (0 == module_categories_.cannot_be_required.count(id)) {
+      module_categories_.can_be_required.emplace(id);
+    }
+  }
+
+  module_categories_.is_initialized = true;
+}
+
 // TODO(joyeecheung): make these more general and put them into util.h
 Local<Object> MapToObject(Local<Context> context,
                           const NativeModuleRecordMap& in) {
@@ -63,6 +132,39 @@ bool NativeModuleLoader::Exists(const char* id) {
   return source_.find(id) != source_.end();
 }
 
+void NativeModuleLoader::GetModuleCategories(
+    Local<Name> property, const PropertyCallbackInfo<Value>& info) {
+  per_process::native_module_loader.InitializeModuleCategories();
+
+  Environment* env = Environment::GetCurrent(info);
+  Isolate* isolate = env->isolate();
+  Local<Context> context = env->context();
+  Local<Object> result = Object::New(isolate);
+
+  // Copy from the per-process categories
+  std::set<std::string> cannot_be_required =
+      per_process::native_module_loader.module_categories_.cannot_be_required;
+  std::set<std::string> can_be_required =
+      per_process::native_module_loader.module_categories_.can_be_required;
+
+  if (!env->owns_process_state()) {
+    can_be_required.erase("trace_events");
+    cannot_be_required.insert("trace_events");
+  }
+
+  result
+      ->Set(context,
+            OneByteString(isolate, "cannotBeRequired"),
+            ToJsSet(context, cannot_be_required))
+      .FromJust();
+  result
+      ->Set(context,
+            OneByteString(isolate, "canBeRequired"),
+            ToJsSet(context, can_be_required))
+      .FromJust();
+  info.GetReturnValue().Set(result);
+}
+
 void NativeModuleLoader::GetCacheUsage(
     const FunctionCallbackInfo<Value>& args) {
   Environment* env = Environment::GetCurrent(args);
@@ -298,6 +400,18 @@ void NativeModuleLoader::Initialize(Local<Object> target,
                           SideEffectType::kHasNoSideEffect)
             .FromJust());
 
+  CHECK(target
+            ->SetAccessor(
+                env->context(),
+                FIXED_ONE_BYTE_STRING(env->isolate(), "moduleCategories"),
+                GetModuleCategories,
+                nullptr,
+                env->as_callback_data(),
+                DEFAULT,
+                None,
+                SideEffectType::kHasNoSideEffect)
+            .FromJust());
+
   env->SetMethod(
       target, "getCacheUsage", NativeModuleLoader::GetCacheUsage);
   env->SetMethod(

src/node_native_module.h

@@ -54,6 +54,9 @@ class NativeModuleLoader {
       Environment* optional_env);
 
  private:
+  static void GetModuleCategories(
+      v8::Local<v8::Name> property,
+      const v8::PropertyCallbackInfo<v8::Value>& info);
   static void GetCacheUsage(const v8::FunctionCallbackInfo<v8::Value>& args);
   // Passing ids of builtin module source code into JS land as
   // internalBinding('native_module').moduleIds
@@ -84,6 +87,15 @@ class NativeModuleLoader {
   static v8::MaybeLocal<v8::Function> CompileAsModule(Environment* env,
                                                       const char* id);
 
+  void InitializeModuleCategories();
+  struct ModuleCategories {
+    bool is_initialized = false;
+    std::set<std::string> can_be_required;
+    std::set<std::string> cannot_be_required;
+  };
+
+  ModuleCategories module_categories_;
+
   NativeModuleRecordMap source_;
   NativeModuleCacheMap code_cache_;
   UnionBytes config_;

test/code-cache/test-code-cache.js

@@ -7,22 +7,15 @@
 
 const { isMainThread } = require('../common');
 const assert = require('assert');
-const {
-  cachableBuiltins,
-  cannotBeRequired
-} = require('internal/bootstrap/cache');
-
 const {
   internalBinding
 } = require('internal/test/binding');
 const {
-  getCacheUsage
+  getCacheUsage,
+  moduleCategories: { canBeRequired, cannotBeRequired }
 } = internalBinding('native_module');
 
-for (const key of cachableBuiltins) {
+for (const key of canBeRequired) {
-  if (!isMainThread && key === 'trace_events') {
-    continue;  // Cannot load trace_events in workers
-  }
   require(key);
 }
 
@@ -60,7 +53,7 @@ if (process.config.variables.node_code_cache_path === undefined) {
   );
 
   for (const key of loadedModules) {
-    if (cannotBeRequired.includes(key)) {
+    if (cannotBeRequired.has(key)) {
       assert(compiledWithoutCache.has(key),
              `"${key}" should've been compiled without code cache`);
     } else {

tools/generate_code_cache.js

@@ -7,11 +7,12 @@
 // compiled into the binary using the `--code-cache-path` option
 // of `configure`.
 
+const { internalBinding } = require('internal/test/binding');
 const {
+  moduleCategories: { canBeRequired },
   getCodeCache,
   compileFunction,
-  cachableBuiltins
+} = internalBinding('native_module');
-} = require('internal/bootstrap/cache');
 
 const {
   types: {
@@ -85,7 +86,9 @@ function lexical(a, b) {
   return 0;
 }
 
-for (const key of cachableBuiltins.sort(lexical)) {
+// TODO(joyeecheung): support non-modules that require different
+// parameters in the wrapper.
+for (const key of [...canBeRequired].sort(lexical)) {
   compileFunction(key);  // compile it
   const cachedData = getCodeCache(key);
   if (!isUint8Array(cachedData)) {