tools: implement mkcodecache as an executable

This patch implement a mkcodecache executable on top of the
`NativeModuleLoader` singleton.
This makes it possible to build a Node.js binary with embedded
code cache without building itself using the code cache stub -
the cache is now initialized by `NativeModuleEnv` instead which
can be refactored out of the mkcodecache dependencies.

PR-URL: https://github.com/nodejs/node/pull/27161
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>

Makefile

@@ -115,7 +115,7 @@ with-code-cache:
 	$(PYTHON) ./configure $(CONFIG_FLAGS)
 	$(MAKE)
 	mkdir -p $(CODE_CACHE_DIR)
-	out/$(BUILDTYPE)/$(NODE_EXE) --expose-internals tools/generate_code_cache.js $(CODE_CACHE_FILE)
+	out/$(BUILDTYPE)/mkcodecache $(CODE_CACHE_FILE)
 	$(PYTHON) ./configure --code-cache-path $(CODE_CACHE_FILE) $(CONFIG_FLAGS)
 	$(MAKE)
 
@@ -1232,6 +1232,8 @@ LINT_CPP_FILES = $(filter-out $(LINT_CPP_EXCLUDE), $(wildcard \
 	test/node-api/*/*.h \
 	tools/icu/*.cc \
 	tools/icu/*.h \
+	tools/code_cache/*.cc \
+	tools/code_cache/*.h \
 	))
 
 # Code blocks don't have newline at the end,

node.gyp

@@ -1103,6 +1103,59 @@
         }],
       ],
     }, # cctest
+    # TODO(joyeecheung): do not depend on node_lib,
+    # instead create a smaller static library node_lib_base that does
+    # just enough for node_native_module.cc and the cache builder to
+    # compile without compiling the generated code cache C++ file.
+    # So generate_code_cache -> mkcodecache -> node_lib_base,
+    #    node_lib -> node_lib_base & generate_code_cache
+    {
+      'target_name': 'mkcodecache',
+      'type': 'executable',
+
+      'dependencies': [
+        '<(node_lib_target_name)',
+        'deps/histogram/histogram.gyp:histogram',
+      ],
+
+      'includes': [
+        'node.gypi'
+      ],
+
+      'include_dirs': [
+        'src',
+        'tools/msvs/genfiles',
+        'deps/v8/include',
+        'deps/cares/include',
+        'deps/uv/include',
+      ],
+
+      'defines': [ 'NODE_WANT_INTERNALS=1' ],
+
+      'sources': [
+        'tools/code_cache/mkcodecache.cc',
+        'tools/code_cache/cache_builder.cc'
+      ],
+
+      'conditions': [
+        [ 'node_report=="true"', {
+          'conditions': [
+            ['OS=="win"', {
+              'libraries': [
+                'dbghelp.lib',
+                'PsApi.lib',
+                'Ws2_32.lib',
+              ],
+              'dll_files': [
+                'dbghelp.dll',
+                'PsApi.dll',
+                'Ws2_32.dll',
+              ],
+            }],
+          ],
+        }],
+      ],
+    }, # cache_builder
   ], # end targets
 
   'conditions': [

src/node_native_module_env.cc

@@ -4,7 +4,6 @@
 namespace node {
 namespace native_module {
 
-using v8::ArrayBuffer;
 using v8::Context;
 using v8::DEFAULT;
 using v8::Function;
@@ -18,11 +17,9 @@ using v8::Name;
 using v8::None;
 using v8::Object;
 using v8::PropertyCallbackInfo;
-using v8::ScriptCompiler;
 using v8::Set;
 using v8::SideEffectType;
 using v8::String;
-using v8::Uint8Array;
 using v8::Value;
 
 // TODO(joyeecheung): make these more general and put them into util.h
@@ -154,26 +151,6 @@ MaybeLocal<Function> NativeModuleEnv::LookupAndCompile(
   return maybe;
 }
 
-// This is supposed to be run only by the main thread in
-// tools/generate_code_cache.js
-void NativeModuleEnv::GetCodeCache(const FunctionCallbackInfo<Value>& args) {
-  Environment* env = Environment::GetCurrent(args);
-  Isolate* isolate = env->isolate();
-  CHECK(env->is_main_thread());
-
-  CHECK(args[0]->IsString());
-  node::Utf8Value id_v(isolate, args[0].As<String>());
-  const char* id = *id_v;
-
-  ScriptCompiler::CachedData* cached_data =
-      NativeModuleLoader::GetInstance()->GetCodeCache(id);
-  if (cached_data != nullptr) {
-    Local<ArrayBuffer> buf = ArrayBuffer::New(isolate, cached_data->length);
-    memcpy(buf->GetContents().Data(), cached_data->data, cached_data->length);
-    args.GetReturnValue().Set(Uint8Array::New(buf, 0, cached_data->length));
-  }
-}
-
 // TODO(joyeecheung): It is somewhat confusing that Class::Initialize
 // is used to initilaize to the binding, but it is the current convention.
 // Rename this across the code base to something that makes more sense.
@@ -216,7 +193,6 @@ void NativeModuleEnv::Initialize(Local<Object> target,
       .Check();
 
   env->SetMethod(target, "getCacheUsage", NativeModuleEnv::GetCacheUsage);
-  env->SetMethod(target, "getCodeCache", NativeModuleEnv::GetCodeCache);
   env->SetMethod(target, "compileFunction", NativeModuleEnv::CompileFunction);
   // internalBinding('native_module') should be frozen
   target->SetIntegrityLevel(context, IntegrityLevel::kFrozen).FromJust();

src/node_native_module_env.h

@@ -52,7 +52,6 @@ class NativeModuleEnv {
       const v8::PropertyCallbackInfo<v8::Value>& info);
   // Compile a specific native module as a function
   static void CompileFunction(const v8::FunctionCallbackInfo<v8::Value>& args);
-  static void GetCodeCache(const v8::FunctionCallbackInfo<v8::Value>& args);
 };
 
 }  // namespace native_module

test/code-cache/test-code-cache-generator.js

@@ -1,10 +1,10 @@
 'use strict';
 
-// This test verifies that the binary is compiled with code cache and the
-// cache is used when built in modules are compiled.
+// This test verifies the code cache generator can generate a C++
+// file that contains the code cache. This can be removed once we
+// actually build that C++ file into our binary.
 
 const common = require('../common');
-
 const tmpdir = require('../common/tmpdir');
 const { spawnSync } = require('child_process');
 const assert = require('assert');
@@ -12,17 +12,29 @@ const path = require('path');
 const fs = require('fs');
 const readline = require('readline');
 
-const generator = path.join(
-  __dirname, '..', '..', 'tools', 'generate_code_cache.js'
-);
+console.log('Looking for mkcodecache executable');
+let buildDir;
+const stat = fs.statSync(process.execPath);
+if (stat.isSymbolicLink()) {
+  console.log('Binary is a symbolic link');
+  buildDir = path.dirname(fs.readlinkSync(process.execPath));
+} else {
+  buildDir = path.dirname(process.execPath);
+}
+
+const ext = common.isWindows ? '.exe' : '';
+const generator = path.join(buildDir, `mkcodecache${ext}`);
+if (!fs.existsSync(generator)) {
+  common.skip('Could not find mkcodecache');
+}
+
+console.log(`mkcodecache is ${generator}`);
+
 tmpdir.refresh();
 const dest = path.join(tmpdir.path, 'cache.cc');
 
-// Run tools/generate_code_cache.js
-const child = spawnSync(
-  process.execPath,
-  ['--expose-internals', generator, dest]
-);
+// Run mkcodecache
+const child = spawnSync(generator, [dest]);
 assert.ifError(child.error);
 if (child.status !== 0) {
   console.log(child.stderr.toString());

tools/code_cache/cache_builder.cc

@@ -0,0 +1,165 @@
+#include "cache_builder.h"
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <vector>
+#include <cstdlib>
+#include "util.h"
+
+#include "node_native_module.h"
+
+namespace node {
+namespace native_module {
+
+using v8::Context;
+using v8::Function;
+using v8::Isolate;
+using v8::Local;
+using v8::MaybeLocal;
+using v8::ScriptCompiler;
+
+static std::string GetDefName(const std::string& id) {
+  char buf[64] = {0};
+  size_t size = id.size();
+  CHECK_LT(size, sizeof(buf));
+  for (size_t i = 0; i < size; ++i) {
+    char ch = id[i];
+    buf[i] = (ch == '-' || ch == '/') ? '_' : ch;
+  }
+  return buf;
+}
+
+static std::string FormatSize(size_t size) {
+  char buf[64] = {0};
+  if (size < 1024) {
+    snprintf(buf, sizeof(buf), "%.2fB", static_cast<double>(size));
+  } else if (size < 1024 * 1024) {
+    snprintf(buf, sizeof(buf), "%.2fKB", static_cast<double>(size / 1024));
+  } else {
+    snprintf(
+        buf, sizeof(buf), "%.2fMB", static_cast<double>(size / 1024 / 1024));
+  }
+  return buf;
+}
+
+static std::string GetDefinition(const std::string& id,
+                                 size_t size,
+                                 const uint8_t* data) {
+  std::stringstream ss;
+  ss << "static const uint8_t " << GetDefName(id) << "[] = {\n";
+  for (size_t i = 0; i < size; ++i) {
+    uint8_t ch = data[i];
+    ss << std::to_string(ch) << (i == size - 1 ? '\n' : ',');
+  }
+  ss << "};";
+  return ss.str();
+}
+
+static std::string GetInitializer(const std::string& id) {
+  std::string def_name = GetDefName(id);
+  char buf[256] = {0};
+  snprintf(buf,
+           sizeof(buf),
+           "code_cache->emplace(\n"
+           "  \"%s\",\n"
+           "  std::make_unique<v8::ScriptCompiler::CachedData>"
+           "(%s, static_cast<int>(arraysize(%s)), policy)\n"
+           ");",
+           id.c_str(),
+           def_name.c_str(),
+           def_name.c_str());
+  return buf;
+}
+
+static std::string GenerateCodeCache(
+    std::map<std::string, ScriptCompiler::CachedData*> data,
+    std::vector<std::string> ids,
+    bool log_progress) {
+  std::stringstream ss;
+  ss << R"(#include <cinttypes>
+#include "node_native_module_env.h"
+
+// This file is generated by tools/mkcodecache
+// and is used when configure is run with \`--code-cache-path\`
+
+namespace node {
+namespace native_module {
+)";
+
+  size_t total = 0;
+  for (const auto& x : data) {
+    const std::string& id = x.first;
+    ScriptCompiler::CachedData* cached_data = x.second;
+    total += cached_data->length;
+    std::string def = GetDefinition(id, cached_data->length, cached_data->data);
+    ss << def << "\n\n";
+    if (log_progress) {
+      std::cout << "Generated cache for " << id
+                << ", size = " << FormatSize(cached_data->length)
+                << ", total = " << FormatSize(total) << "\n";
+    }
+  }
+
+  ss << R"(void NativeModuleEnv::InitializeCodeCache() {
+  NativeModuleCacheMap* code_cache =
+      NativeModuleLoader::GetInstance()->code_cache();
+  if (!code_cache->empty()) {
+    return;
+  }
+  auto policy = v8::ScriptCompiler::CachedData::BufferPolicy::BufferNotOwned;
+)";
+
+  for (const auto& x : data) {
+    const std::string& id = x.first;
+    ss << GetInitializer(id) << "\n\n";
+  }
+
+  ss << R"(}
+
+}  // namespace native_module
+}  // namespace node
+)";
+  return ss.str();
+}
+
+std::string CodeCacheBuilder::Generate(Local<Context> context) {
+  NativeModuleLoader* loader = NativeModuleLoader::GetInstance();
+  std::vector<std::string> ids = loader->GetModuleIds();
+
+  std::vector<std::string> modules;
+  modules.reserve(ids.size());
+
+  std::map<std::string, ScriptCompiler::CachedData*> data;
+
+  NativeModuleLoader::Result result;
+  for (const auto& id : ids) {
+    // TODO(joyeecheung): we can only compile the modules that can be
+    // required here because the parameters for other types of builtins
+    // are still very flexible. We should look into auto-generating
+    // the paramters from the source somehow.
+    if (loader->CanBeRequired(id.c_str())) {
+      modules.push_back(id);
+      USE(loader->CompileAsModule(context, id.c_str(), &result));
+      ScriptCompiler::CachedData* cached_data =
+          loader->GetCodeCache(id.c_str());
+      if (cached_data == nullptr) {
+        // TODO(joyeecheung): display syntax errors
+        std::cerr << "Failed to complile " << id << "\n";
+      } else {
+        data.emplace(id, cached_data);
+      }
+    }
+  }
+
+  char env_buf[32];
+  size_t env_size = sizeof(env_buf);
+  int ret = uv_os_getenv("NODE_DEBUG", env_buf, &env_size);
+  bool log_progress = false;
+  if (ret == 0 && strcmp(env_buf, "mkcodecache") == 0) {
+    log_progress = true;
+  }
+  return GenerateCodeCache(data, modules, log_progress);
+}
+
+}  // namespace native_module
+}  // namespace node

tools/code_cache/cache_builder.h

@@ -0,0 +1,16 @@
+#ifndef TOOLS_CODE_CACHE_CACHE_BUILDER_H_
+#define TOOLS_CODE_CACHE_CACHE_BUILDER_H_
+
+#include <string>
+#include "v8.h"
+
+namespace node {
+namespace native_module {
+class CodeCacheBuilder {
+ public:
+  static std::string Generate(v8::Local<v8::Context> context);
+};
+}  // namespace native_module
+}  // namespace node
+
+#endif  // TOOLS_CODE_CACHE_CACHE_BUILDER_H_

tools/code_cache/mkcodecache.cc

@@ -0,0 +1,62 @@
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "cache_builder.h"
+#include "libplatform/libplatform.h"
+#include "v8.h"
+
+using node::native_module::CodeCacheBuilder;
+using v8::ArrayBuffer;
+using v8::Context;
+using v8::HandleScope;
+using v8::Isolate;
+using v8::Local;
+
+#ifdef _WIN32
+#include <VersionHelpers.h>
+#include <WinError.h>
+#include <windows.h>
+
+int wmain(int argc, wchar_t* argv[]) {
+#else   // UNIX
+int main(int argc, char* argv[]) {
+#endif  // _WIN32
+
+  if (argc < 2) {
+    std::cerr << "Usage: " << argv[0] << " <path/to/output.cc>\n";
+    return 1;
+  }
+
+  std::ofstream out;
+  out.open(argv[1], std::ios::out | std::ios::binary);
+  if (!out.is_open()) {
+    std::cerr << "Cannot open " << argv[1] << "\n";
+    return 1;
+  }
+
+  std::unique_ptr<v8::Platform> platform = v8::platform::NewDefaultPlatform();
+  v8::V8::InitializePlatform(platform.get());
+  v8::V8::Initialize();
+
+  // Create a new Isolate and make it the current one.
+  Isolate::CreateParams create_params;
+  create_params.array_buffer_allocator =
+      ArrayBuffer::Allocator::NewDefaultAllocator();
+  Isolate* isolate = Isolate::New(create_params);
+  {
+    Isolate::Scope isolate_scope(isolate);
+    v8::HandleScope handle_scope(isolate);
+    v8::Local<v8::Context> context = v8::Context::New(isolate);
+    v8::Context::Scope context_scope(context);
+
+    std::string cache = CodeCacheBuilder::Generate(context);
+    out << cache;
+    out.close();
+  }
+
+  return 0;
+}

tools/generate_code_cache.js

@@ -1,135 +0,0 @@
-'use strict';
-
-// Flags: --expose-internals
-
-// This file generates the code cache for builtin modules and
-// writes them into static char arrays of a C++ file that can be
-// compiled into the binary using the `--code-cache-path` option
-// of `configure`.
-
-const { internalBinding } = require('internal/test/binding');
-const {
-  moduleCategories: { canBeRequired },
-  getCodeCache,
-  compileFunction,
-} = internalBinding('native_module');
-
-const {
-  types: {
-    isUint8Array
-  }
-} = require('util');
-
-const fs = require('fs');
-
-const resultPath = process.argv[2];
-if (!resultPath) {
-  console.error(`Usage: ${process.argv[0]} ${process.argv[1]}` +
-                'path/to/node_code_cache.cc');
-  process.exit(1);
-}
-
-/**
- * Format a number of a size in bytes into human-readable strings
- * @param {number} num
- * @return {string}
- */
-function formatSize(num) {
-  if (num < 1024) {
-    return `${(num).toFixed(2)}B`;
-  } else if (num < 1024 ** 2) {
-    return `${(num / 1024).toFixed(2)}KB`;
-  } else if (num < 1024 ** 3) {
-    return `${(num / (1024 ** 2)).toFixed(2)}MB`;
-  } else {
-    return `${(num / (1024 ** 3)).toFixed(2)}GB`;
-  }
-}
-
-/**
- * Generates the source code of definitions of the char arrays
- * that contains the code cache and the source code of the
- * initializers of the code cache.
- *
- * @param {string} key ID of the builtin module
- * @param {Uint8Array} cache Code cache of the builtin module
- * @return { definition: string, initializer: string }
- */
-function getInitalizer(key, cache) {
-  const defName = `${key.replace(/\//g, '_').replace(/-/g, '_')}_raw`;
-  const definition = `static const uint8_t ${defName}[] = {\n` +
-                     `${cache.join(',')}\n};`;
-  const dataDef = 'std::make_unique<v8::ScriptCompiler::CachedData>(' +
-                  `${defName}, static_cast<int>(arraysize(${defName})), ` +
-                  'policy)';
-  const initializer =
-    'code_cache->emplace(\n' +
-    `  "${key}",\n` +
-    `  ${dataDef}\n` +
-    ');';
-  return {
-    definition, initializer
-  };
-}
-
-const cacheDefinitions = [];
-const cacheInitializers = [];
-let totalCacheSize = 0;
-
-function lexical(a, b) {
-  if (a < b) {
-    return -1;
-  }
-  if (a > b) {
-    return 1;
-  }
-  return 0;
-}
-
-// TODO(joyeecheung): support non-modules that require different
-// parameters in the wrapper.
-for (const key of [...canBeRequired].sort(lexical)) {
-  compileFunction(key);  // compile it
-  const cachedData = getCodeCache(key);
-  if (!isUint8Array(cachedData)) {
-    console.error(`Failed to generate code cache for '${key}'`);
-    process.exit(1);
-  }
-
-  const size = cachedData.byteLength;
-  totalCacheSize += size;
-  const {
-    definition, initializer,
-  } = getInitalizer(key, cachedData);
-  cacheDefinitions.push(definition);
-  cacheInitializers.push(initializer);
-  console.log(`Generated cache for '${key}', size = ${formatSize(size)}` +
-              `, total = ${formatSize(totalCacheSize)}`);
-}
-
-const result = `#include "node_native_module_env.h"
-
-// This file is generated by tools/generate_code_cache.js
-// and is used when configure is run with \`--code-cache-path\`
-
-namespace node {
-namespace native_module {
-${cacheDefinitions.join('\n\n')}
-
-void NativeModuleEnv::InitializeCodeCache() {
-  NativeModuleCacheMap* code_cache =
-      NativeModuleLoader::GetInstance()->code_cache();
-  if (!code_cache->empty()) {
-    return;
-  }
-
-  auto policy = v8::ScriptCompiler::CachedData::BufferPolicy::BufferNotOwned;
-  ${cacheInitializers.join('\n  ')}
-}
-
-}  // namespace native_module
-}  // namespace node
-`;
-
-fs.writeFileSync(resultPath, result);
-console.log(`Generated code cache C++ file to ${resultPath}`);