1berry/nodejs
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

src: compile native modules and their code cache in C++

Browse Source 
This patch refactors out a part of NativeModule.prototype.compile
(in JS land) into a C++ NativeModule class, this enables a
couple of possibilities:

1. By moving the code to the C++ land, we have more opportunity
  to specialize the compilation process of the native modules
  (e.g. compilation options, code cache) that is orthogonal to
  how user land modules are compiled
2. We can reuse the code to compile bootstrappers and context
  fixers and enable them to be compiled with the code cache later,
  since they are not loaded by NativeModule in the JS land their
  caching must be done in C++.
3. Since there is no need to pass the static data to JS for
  compilation anymore, this enables us to use
  (std::map<std::string, const char*>) in the generated
  node_code_cache.cc and node_javascript.cc later, and scope
  every actual access to the source of native modules to a
  std::map lookup instead of a lookup on a v8::Object in
  dictionary mode.

This patch also refactor the code cache generator and tests
a bit and trace the `withCodeCache` and `withoutCodeCache`
in a Set instead of an Array, and makes sure that all the cachable
builtins are tested.

PR-URL: https://github.com/nodejs/node/pull/24221
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: Anna Henningsen <anna@addaleax.net>
This commit is contained in:
Joyee Cheung 2018-11-03 14:26:32 +08:00
parent 3856d8a14c
commit bd765d61d7
No known key found for this signature in database
GPG Key ID: 92B78A53C8303B8D
12 changed files with 680 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
lib/internal/bootstrap/cache.js
View File

@ -8,20 +8,13 @@
const {
NativeModule
} = require('internal/bootstrap/loaders');
const {
source,
compileCodeCache
} = internalBinding('native_module');
const { hasTracing } = process.binding('config');
function getCodeCache(id) {
const cached = NativeModule.getCached(id);
if (cached && (cached.loaded || cached.loading)) {
return cached.script.createCachedData();
}
// The script has not been compiled and run
NativeModule.require(id);
return getCodeCache(id);
}
const depsModule = Object.keys(NativeModule._source).filter(
const depsModule = Object.keys(source).filter(
(key) => NativeModule.isDepsModule(key) || key.startsWith('internal/deps')
);
@ -75,17 +68,10 @@ if (!process.versions.openssl) {
}
module.exports = {
cachableBuiltins: Object.keys(NativeModule._source).filter(
cachableBuiltins: Object.keys(source).filter(
(key) => !cannotUseCache.includes(key)
),
builtinSource: Object.assign({}, NativeModule._source),
getCodeCache,
getSource: NativeModule.getSource,
codeCache: internalBinding('code_cache'),
compiledWithoutCache: NativeModule.compiledWithoutCache,
compiledWithCache: NativeModule.compiledWithCache,
nativeModuleWrap(script) {
return NativeModule.wrap(script);
},
getSource(id) { return source[id]; },
getCodeCache: compileCodeCache,
cannotUseCache
};

63
lib/internal/bootstrap/loaders.js
View File

@ -149,7 +149,6 @@
// Create this WeakMap in js-land because V8 has no C++ API for WeakMap
internalBinding('module_wrap').callbackMap = new WeakMap();
const { ContextifyScript } = internalBinding('contextify');
// Set up NativeModule
function NativeModule(id) {
@ -160,7 +159,6 @@
this.exportKeys = undefined;
this.loaded = false;
this.loading = false;
this.script = null; // The ContextifyScript of the module
}
NativeModule._source = getInternalBinding('natives');
@ -168,12 +166,6 @@
const config = getBinding('config');
const codeCache = getInternalBinding('code_cache');
const codeCacheHash = getInternalBinding('code_cache_hash');
const sourceHash = getInternalBinding('natives_hash');
const compiledWithoutCache = NativeModule.compiledWithoutCache = [];
const compiledWithCache = NativeModule.compiledWithCache = [];
// Think of this as module.exports in this file even though it is not
// written in CommonJS style.
const loaderExports = { internalBinding, NativeModule };
@ -332,67 +324,18 @@
this.exports = new Proxy(this.exports, handler);
};
const { compileFunction } = getInternalBinding('native_module');
NativeModule.prototype.compile = function() {
const id = this.id;
let source = NativeModule.getSource(id);
source = NativeModule.wrap(source);
this.loading = true;
try {
// Currently V8 only checks that the length of the source code is the
// same as the code used to generate the hash, so we add an additional
// check here:
// 1. During compile time, when generating node_javascript.cc and
// node_code_cache.cc, we compute and include the hash of the
// (unwrapped) JavaScript source in both.
// 2. At runtime, we check that the hash of the code being compiled
// and the hash of the code used to generate the cache
// (inside the wrapper) is the same.
// This is based on the assumptions:
// 1. `internalBinding('code_cache_hash')` must be in sync with
// `internalBinding('code_cache')` (same C++ file)
// 2. `internalBinding('natives_hash')` must be in sync with
// `internalBinding('natives')` (same C++ file)
// 3. If `internalBinding('natives_hash')` is in sync with
// `internalBinding('natives_hash')`, then the (unwrapped)
// code used to generate `internalBinding('code_cache')`
// should be in sync with the (unwrapped) code in
// `internalBinding('natives')`
// There will be, however, false positives if the wrapper used
// to generate the cache is different from the one used at run time,
// and the length of the wrapper somehow stays the same.
// But that should be rare and can be eased once we make the
// two bootstrappers cached and checked as well.
const cache = codeCacheHash[id] &&
(codeCacheHash[id] === sourceHash[id]) ? codeCache[id] : undefined;
// (code, filename, lineOffset, columnOffset
// cachedData, produceCachedData, parsingContext)
const script = new ContextifyScript(
source, this.filename, 0, 0,
cache, false, undefined
);
// This will be used to create code cache in tools/generate_code_cache.js
this.script = script;
// One of these conditions may be false when any of the inputs
// of the `node_js2c` target in node.gyp is modified.
// FIXME(joyeecheung): Figure out how to resolve the dependency issue.
// When the code cache was introduced we were at a point where refactoring
// node.gyp may not be worth the effort.
if (!cache || script.cachedDataRejected) {
compiledWithoutCache.push(this.id);
} else {
compiledWithCache.push(this.id);
}
// Arguments: timeout, displayErrors, breakOnSigint
const fn = script.runInThisContext(-1, true, false);
const requireFn = this.id.startsWith('internal/deps/') ?
NativeModule.requireForDeps :
NativeModule.require;
const fn = compileFunction(id);
fn(this.exports, requireFn, this, process, internalBinding);
if (config.experimentalModules && !NativeModule.isInternal(this.id)) {

2
node.gyp
View File

@ -416,6 +416,8 @@
'src/node_javascript.h',
'src/node_messaging.h',
'src/node_mutex.h',
'src/node_native_module.h',
'src/node_native_module.cc',
'src/node_options.h',
'src/node_options-inl.h',
'src/node_perf.h',

490
src/env.h
View File

@ -120,248 +120,258 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2;
// Strings are per-isolate primitives but Environment proxies them
// for the sake of convenience. Strings should be ASCII-only.
#define PER_ISOLATE_STRING_PROPERTIES(V) \
V(address_string, "address") \
V(aliases_string, "aliases") \
V(args_string, "args") \
V(async_ids_stack_string, "async_ids_stack") \
V(buffer_string, "buffer") \
V(bytes_parsed_string, "bytesParsed") \
V(bytes_read_string, "bytesRead") \
V(bytes_written_string, "bytesWritten") \
V(cached_data_string, "cachedData") \
V(cached_data_produced_string, "cachedDataProduced") \
V(cached_data_rejected_string, "cachedDataRejected") \
V(change_string, "change") \
V(channel_string, "channel") \
V(chunks_sent_since_last_write_string, "chunksSentSinceLastWrite") \
V(constants_string, "constants") \
V(oncertcb_string, "oncertcb") \
V(code_string, "code") \
V(cwd_string, "cwd") \
V(dest_string, "dest") \
V(destroyed_string, "destroyed") \
V(detached_string, "detached") \
V(dns_a_string, "A") \
V(dns_aaaa_string, "AAAA") \
V(dns_cname_string, "CNAME") \
V(dns_mx_string, "MX") \
V(dns_naptr_string, "NAPTR") \
V(dns_ns_string, "NS") \
V(dns_ptr_string, "PTR") \
V(dns_soa_string, "SOA") \
V(dns_srv_string, "SRV") \
V(dns_txt_string, "TXT") \
V(duration_string, "duration") \
V(emit_warning_string, "emitWarning") \
V(exchange_string, "exchange") \
V(encoding_string, "encoding") \
V(entries_string, "entries") \
V(entry_type_string, "entryType") \
V(env_pairs_string, "envPairs") \
V(env_var_settings_string, "envVarSettings") \
V(errno_string, "errno") \
V(error_string, "error") \
V(exit_code_string, "exitCode") \
V(expire_string, "expire") \
V(exponent_string, "exponent") \
V(exports_string, "exports") \
V(ext_key_usage_string, "ext_key_usage") \
V(external_stream_string, "_externalStream") \
V(family_string, "family") \
V(fatal_exception_string, "_fatalException") \
V(fd_string, "fd") \
V(file_string, "file") \
V(fingerprint_string, "fingerprint") \
V(fingerprint256_string, "fingerprint256") \
V(flags_string, "flags") \
V(fragment_string, "fragment") \
V(get_data_clone_error_string, "_getDataCloneError") \
V(get_shared_array_buffer_id_string, "_getSharedArrayBufferId") \
V(gid_string, "gid") \
V(handle_string, "handle") \
V(help_text_string, "helpText") \
V(homedir_string, "homedir") \
V(host_string, "host") \
V(hostmaster_string, "hostmaster") \
V(ignore_string, "ignore") \
V(infoaccess_string, "infoAccess") \
V(inherit_string, "inherit") \
V(input_string, "input") \
V(internal_string, "internal") \
V(ipv4_string, "IPv4") \
V(ipv6_string, "IPv6") \
V(isclosing_string, "isClosing") \
V(issuer_string, "issuer") \
V(issuercert_string, "issuerCertificate") \
V(kill_signal_string, "killSignal") \
V(kind_string, "kind") \
V(mac_string, "mac") \
V(main_string, "main") \
V(max_buffer_string, "maxBuffer") \
V(message_string, "message") \
V(message_port_string, "messagePort") \
V(message_port_constructor_string, "MessagePort") \
V(minttl_string, "minttl") \
V(modulus_string, "modulus") \
V(name_string, "name") \
V(netmask_string, "netmask") \
V(nsname_string, "nsname") \
V(ocsp_request_string, "OCSPRequest") \
V(onaltsvc_string, "onaltsvc") \
V(onchange_string, "onchange") \
V(onclienthello_string, "onclienthello") \
V(oncomplete_string, "oncomplete") \
V(onconnection_string, "onconnection") \
V(ondone_string, "ondone") \
V(onerror_string, "onerror") \
V(onexit_string, "onexit") \
V(onframeerror_string, "onframeerror") \
V(ongetpadding_string, "ongetpadding") \
V(onhandshakedone_string, "onhandshakedone") \
V(onhandshakestart_string, "onhandshakestart") \
V(onheaders_string, "onheaders") \
V(onmessage_string, "onmessage") \
V(onnewsession_string, "onnewsession") \
V(onocspresponse_string, "onocspresponse") \
V(ongoawaydata_string, "ongoawaydata") \
V(onorigin_string, "onorigin") \
V(onpriority_string, "onpriority") \
V(onread_string, "onread") \
V(onreadstart_string, "onreadstart") \
V(onreadstop_string, "onreadstop") \
V(onping_string, "onping") \
V(onsettings_string, "onsettings") \
V(onshutdown_string, "onshutdown") \
V(onsignal_string, "onsignal") \
V(onstreamclose_string, "onstreamclose") \
V(ontrailers_string, "ontrailers") \
V(onunpipe_string, "onunpipe") \
V(onwrite_string, "onwrite") \
V(openssl_error_stack, "opensslErrorStack") \
V(options_string, "options") \
V(output_string, "output") \
V(order_string, "order") \
V(parse_error_string, "Parse Error") \
V(password_string, "password") \
V(path_string, "path") \
V(pending_handle_string, "pendingHandle") \
V(pid_string, "pid") \
V(pipe_string, "pipe") \
V(pipe_target_string, "pipeTarget") \
V(pipe_source_string, "pipeSource") \
V(port_string, "port") \
V(port1_string, "port1") \
V(port2_string, "port2") \
V(preference_string, "preference") \
V(priority_string, "priority") \
V(promise_string, "promise") \
V(pubkey_string, "pubkey") \
V(query_string, "query") \
V(raw_string, "raw") \
V(read_host_object_string, "_readHostObject") \
V(readable_string, "readable") \
V(reason_string, "reason") \
V(refresh_string, "refresh") \
V(regexp_string, "regexp") \
V(rename_string, "rename") \
V(replacement_string, "replacement") \
V(retry_string, "retry") \
V(scheme_string, "scheme") \
V(serial_string, "serial") \
V(scopeid_string, "scopeid") \
V(serial_number_string, "serialNumber") \
V(service_string, "service") \
V(servername_string, "servername") \
V(session_id_string, "sessionId") \
V(shell_string, "shell") \
V(signal_string, "signal") \
V(sink_string, "sink") \
V(size_string, "size") \
V(sni_context_err_string, "Invalid SNI context") \
V(sni_context_string, "sni_context") \
V(source_string, "source") \
V(stack_string, "stack") \
V(start_time_string, "startTime") \
V(status_string, "status") \
V(stdio_string, "stdio") \
V(subject_string, "subject") \
V(subjectaltname_string, "subjectaltname") \
V(syscall_string, "syscall") \
V(thread_id_string, "threadId") \
V(ticketkeycallback_string, "onticketkeycallback") \
V(timeout_string, "timeout") \
V(tls_ticket_string, "tlsTicket") \
V(ttl_string, "ttl") \
V(type_string, "type") \
V(uid_string, "uid") \
V(unknown_string, "<unknown>") \
V(url_string, "url") \
V(username_string, "username") \
V(valid_from_string, "valid_from") \
V(valid_to_string, "valid_to") \
V(value_string, "value") \
V(verify_error_string, "verifyError") \
V(version_string, "version") \
V(weight_string, "weight") \
V(windows_hide_string, "windowsHide") \
V(windows_verbatim_arguments_string, "windowsVerbatimArguments") \
V(wrap_string, "wrap") \
V(writable_string, "writable") \
V(write_host_object_string, "_writeHostObject") \
V(write_queue_size_string, "writeQueueSize") \
V(x_forwarded_string, "x-forwarded-for") \
#define PER_ISOLATE_STRING_PROPERTIES(V) \
V(address_string, "address") \
V(aliases_string, "aliases") \
V(args_string, "args") \
V(async_ids_stack_string, "async_ids_stack") \
V(buffer_string, "buffer") \
V(bytes_parsed_string, "bytesParsed") \
V(bytes_read_string, "bytesRead") \
V(bytes_written_string, "bytesWritten") \
V(cached_data_string, "cachedData") \
V(cached_data_produced_string, "cachedDataProduced") \
V(cached_data_rejected_string, "cachedDataRejected") \
V(change_string, "change") \
V(channel_string, "channel") \
V(chunks_sent_since_last_write_string, "chunksSentSinceLastWrite") \
V(constants_string, "constants") \
V(oncertcb_string, "oncertcb") \
V(code_string, "code") \
V(cwd_string, "cwd") \
V(dest_string, "dest") \
V(destroyed_string, "destroyed") \
V(detached_string, "detached") \
V(dns_a_string, "A") \
V(dns_aaaa_string, "AAAA") \
V(dns_cname_string, "CNAME") \
V(dns_mx_string, "MX") \
V(dns_naptr_string, "NAPTR") \
V(dns_ns_string, "NS") \
V(dns_ptr_string, "PTR") \
V(dns_soa_string, "SOA") \
V(dns_srv_string, "SRV") \
V(dns_txt_string, "TXT") \
V(duration_string, "duration") \
V(emit_warning_string, "emitWarning") \
V(exchange_string, "exchange") \
V(encoding_string, "encoding") \
V(entries_string, "entries") \
V(entry_type_string, "entryType") \
V(env_pairs_string, "envPairs") \
V(env_var_settings_string, "envVarSettings") \
V(errno_string, "errno") \
V(error_string, "error") \
V(exit_code_string, "exitCode") \
V(expire_string, "expire") \
V(exponent_string, "exponent") \
V(exports_string, "exports") \
V(ext_key_usage_string, "ext_key_usage") \
V(external_stream_string, "_externalStream") \
V(family_string, "family") \
V(fatal_exception_string, "_fatalException") \
V(fd_string, "fd") \
V(file_string, "file") \
V(fingerprint_string, "fingerprint") \
V(fingerprint256_string, "fingerprint256") \
V(flags_string, "flags") \
V(fragment_string, "fragment") \
V(get_data_clone_error_string, "_getDataCloneError") \
V(get_shared_array_buffer_id_string, "_getSharedArrayBufferId") \
V(gid_string, "gid") \
V(handle_string, "handle") \
V(help_text_string, "helpText") \
V(homedir_string, "homedir") \
V(host_string, "host") \
V(hostmaster_string, "hostmaster") \
V(ignore_string, "ignore") \
V(infoaccess_string, "infoAccess") \
V(inherit_string, "inherit") \
V(input_string, "input") \
V(internal_string, "internal") \
V(internal_binding_string, "internalBinding") \
V(ipv4_string, "IPv4") \
V(ipv6_string, "IPv6") \
V(isclosing_string, "isClosing") \
V(issuer_string, "issuer") \
V(issuercert_string, "issuerCertificate") \
V(kill_signal_string, "killSignal") \
V(kind_string, "kind") \
V(mac_string, "mac") \
V(main_string, "main") \
V(max_buffer_string, "maxBuffer") \
V(message_string, "message") \
V(message_port_string, "messagePort") \
V(message_port_constructor_string, "MessagePort") \
V(minttl_string, "minttl") \
V(module_string, "module") \
V(modulus_string, "modulus") \
V(name_string, "name") \
V(netmask_string, "netmask") \
V(nsname_string, "nsname") \
V(ocsp_request_string, "OCSPRequest") \
V(onaltsvc_string, "onaltsvc") \
V(onchange_string, "onchange") \
V(onclienthello_string, "onclienthello") \
V(oncomplete_string, "oncomplete") \
V(onconnection_string, "onconnection") \
V(ondone_string, "ondone") \
V(onerror_string, "onerror") \
V(onexit_string, "onexit") \
V(onframeerror_string, "onframeerror") \
V(ongetpadding_string, "ongetpadding") \
V(onhandshakedone_string, "onhandshakedone") \
V(onhandshakestart_string, "onhandshakestart") \
V(onheaders_string, "onheaders") \
V(onmessage_string, "onmessage") \
V(onnewsession_string, "onnewsession") \
V(onocspresponse_string, "onocspresponse") \
V(ongoawaydata_string, "ongoawaydata") \
V(onorigin_string, "onorigin") \
V(onpriority_string, "onpriority") \
V(onread_string, "onread") \
V(onreadstart_string, "onreadstart") \
V(onreadstop_string, "onreadstop") \
V(onping_string, "onping") \
V(onsettings_string, "onsettings") \
V(onshutdown_string, "onshutdown") \
V(onsignal_string, "onsignal") \
V(onstreamclose_string, "onstreamclose") \
V(ontrailers_string, "ontrailers") \
V(onunpipe_string, "onunpipe") \
V(onwrite_string, "onwrite") \
V(openssl_error_stack, "opensslErrorStack") \
V(options_string, "options") \
V(output_string, "output") \
V(order_string, "order") \
V(parse_error_string, "Parse Error") \
V(password_string, "password") \
V(path_string, "path") \
V(pending_handle_string, "pendingHandle") \
V(pid_string, "pid") \
V(pipe_string, "pipe") \
V(pipe_target_string, "pipeTarget") \
V(pipe_source_string, "pipeSource") \
V(port_string, "port") \
V(port1_string, "port1") \
V(port2_string, "port2") \
V(preference_string, "preference") \
V(priority_string, "priority") \
V(process_string, "process") \
V(promise_string, "promise") \
V(pubkey_string, "pubkey") \
V(query_string, "query") \
V(raw_string, "raw") \
V(read_host_object_string, "_readHostObject") \
V(readable_string, "readable") \
V(reason_string, "reason") \
V(refresh_string, "refresh") \
V(regexp_string, "regexp") \
V(rename_string, "rename") \
V(replacement_string, "replacement") \
V(require_string, "require") \
V(retry_string, "retry") \
V(scheme_string, "scheme") \
V(serial_string, "serial") \
V(scopeid_string, "scopeid") \
V(serial_number_string, "serialNumber") \
V(service_string, "service") \
V(servername_string, "servername") \
V(session_id_string, "sessionId") \
V(shell_string, "shell") \
V(signal_string, "signal") \
V(sink_string, "sink") \
V(size_string, "size") \
V(sni_context_err_string, "Invalid SNI context") \
V(sni_context_string, "sni_context") \
V(source_string, "source") \
V(stack_string, "stack") \
V(start_time_string, "startTime") \
V(status_string, "status") \
V(stdio_string, "stdio") \
V(subject_string, "subject") \
V(subjectaltname_string, "subjectaltname") \
V(syscall_string, "syscall") \
V(thread_id_string, "threadId") \
V(ticketkeycallback_string, "onticketkeycallback") \
V(timeout_string, "timeout") \
V(tls_ticket_string, "tlsTicket") \
V(ttl_string, "ttl") \
V(type_string, "type") \
V(uid_string, "uid") \
V(unknown_string, "<unknown>") \
V(url_string, "url") \
V(username_string, "username") \
V(valid_from_string, "valid_from") \
V(valid_to_string, "valid_to") \
V(value_string, "value") \
V(verify_error_string, "verifyError") \
V(version_string, "version") \
V(weight_string, "weight") \
V(windows_hide_string, "windowsHide") \
V(windows_verbatim_arguments_string, "windowsVerbatimArguments") \
V(wrap_string, "wrap") \
V(writable_string, "writable") \
V(write_host_object_string, "_writeHostObject") \
V(write_queue_size_string, "writeQueueSize") \
V(x_forwarded_string, "x-forwarded-for") \
V(zero_return_string, "ZERO_RETURN")
#define ENVIRONMENT_STRONG_PERSISTENT_PROPERTIES(V) \
V(as_external, v8::External) \
V(async_hooks_after_function, v8::Function) \
V(async_hooks_before_function, v8::Function) \
V(async_hooks_binding, v8::Object) \
V(async_hooks_destroy_function, v8::Function) \
V(async_hooks_init_function, v8::Function) \
V(async_hooks_promise_resolve_function, v8::Function) \
V(async_wrap_object_ctor_template, v8::FunctionTemplate) \
V(async_wrap_ctor_template, v8::FunctionTemplate) \
V(buffer_prototype_object, v8::Object) \
V(context, v8::Context) \
V(domain_callback, v8::Function) \
V(domexception_function, v8::Function) \
V(fdclose_constructor_template, v8::ObjectTemplate) \
V(fd_constructor_template, v8::ObjectTemplate) \
V(filehandlereadwrap_template, v8::ObjectTemplate) \
V(fsreqpromise_constructor_template, v8::ObjectTemplate) \
V(fs_use_promises_symbol, v8::Symbol) \
V(handle_wrap_ctor_template, v8::FunctionTemplate) \
V(host_import_module_dynamically_callback, v8::Function) \
V(host_initialize_import_meta_object_callback, v8::Function) \
V(http2ping_constructor_template, v8::ObjectTemplate) \
V(http2settings_constructor_template, v8::ObjectTemplate) \
V(http2stream_constructor_template, v8::ObjectTemplate) \
V(immediate_callback_function, v8::Function) \
V(inspector_console_api_object, v8::Object) \
V(libuv_stream_wrap_ctor_template, v8::FunctionTemplate) \
V(message_port, v8::Object) \
V(message_port_constructor_template, v8::FunctionTemplate) \
V(pipe_constructor_template, v8::FunctionTemplate) \
V(performance_entry_callback, v8::Function) \
V(performance_entry_template, v8::Function) \
V(process_object, v8::Object) \
V(promise_handler_function, v8::Function) \
V(promise_wrap_template, v8::ObjectTemplate) \
V(sab_lifetimepartner_constructor_template, v8::FunctionTemplate) \
V(script_context_constructor_template, v8::FunctionTemplate) \
V(script_data_constructor_function, v8::Function) \
V(secure_context_constructor_template, v8::FunctionTemplate) \
V(shutdown_wrap_template, v8::ObjectTemplate) \
V(tcp_constructor_template, v8::FunctionTemplate) \
V(tick_callback_function, v8::Function) \
V(timers_callback_function, v8::Function) \
V(tls_wrap_constructor_function, v8::Function) \
V(trace_category_state_function, v8::Function) \
V(tty_constructor_template, v8::FunctionTemplate) \
V(udp_constructor_function, v8::Function) \
V(url_constructor_function, v8::Function) \
#define ENVIRONMENT_STRONG_PERSISTENT_PROPERTIES(V) \
V(as_external, v8::External) \
V(async_hooks_after_function, v8::Function) \
V(async_hooks_before_function, v8::Function) \
V(async_hooks_binding, v8::Object) \
V(async_hooks_destroy_function, v8::Function) \
V(async_hooks_init_function, v8::Function) \
V(async_hooks_promise_resolve_function, v8::Function) \
V(async_wrap_object_ctor_template, v8::FunctionTemplate) \
V(async_wrap_ctor_template, v8::FunctionTemplate) \
V(buffer_prototype_object, v8::Object) \
V(context, v8::Context) \
V(domain_callback, v8::Function) \
V(domexception_function, v8::Function) \
V(fdclose_constructor_template, v8::ObjectTemplate) \
V(fd_constructor_template, v8::ObjectTemplate) \
V(filehandlereadwrap_template, v8::ObjectTemplate) \
V(fsreqpromise_constructor_template, v8::ObjectTemplate) \
V(fs_use_promises_symbol, v8::Symbol) \
V(handle_wrap_ctor_template, v8::FunctionTemplate) \
V(host_import_module_dynamically_callback, v8::Function) \
V(host_initialize_import_meta_object_callback, v8::Function) \
V(http2ping_constructor_template, v8::ObjectTemplate) \
V(http2settings_constructor_template, v8::ObjectTemplate) \
V(http2stream_constructor_template, v8::ObjectTemplate) \
V(immediate_callback_function, v8::Function) \
V(inspector_console_api_object, v8::Object) \
V(libuv_stream_wrap_ctor_template, v8::FunctionTemplate) \
V(message_port, v8::Object) \
V(message_port_constructor_template, v8::FunctionTemplate) \
V(native_modules_code_cache, v8::Object) \
V(native_modules_code_cache_hash, v8::Object) \
V(native_modules_source, v8::Object) \
V(native_modules_source_hash, v8::Object) \
V(native_modules_with_cache, v8::Set) \
V(native_modules_without_cache, v8::Set) \
V(pipe_constructor_template, v8::FunctionTemplate) \
V(performance_entry_callback, v8::Function) \
V(performance_entry_template, v8::Function) \
V(process_object, v8::Object) \
V(promise_handler_function, v8::Function) \
V(promise_wrap_template, v8::ObjectTemplate) \
V(sab_lifetimepartner_constructor_template, v8::FunctionTemplate) \
V(script_context_constructor_template, v8::FunctionTemplate) \
V(script_data_constructor_function, v8::Function) \
V(secure_context_constructor_template, v8::FunctionTemplate) \
V(shutdown_wrap_template, v8::ObjectTemplate) \
V(tcp_constructor_template, v8::FunctionTemplate) \
V(tick_callback_function, v8::Function) \
V(timers_callback_function, v8::Function) \
V(tls_wrap_constructor_function, v8::Function) \
V(trace_category_state_function, v8::Function) \
V(tty_constructor_template, v8::FunctionTemplate) \
V(udp_constructor_function, v8::Function) \
V(url_constructor_function, v8::Function) \
V(write_wrap_template, v8::ObjectTemplate)
class Environment;

27
src/node.cc
View File

@ -25,10 +25,9 @@
#include "node_errors.h"
#include "node_internals.h"
#include "node_javascript.h"
#include "node_code_cache.h"
#include "node_native_module.h"
#include "node_perf.h"
#include "node_platform.h"
#include "node_version.h"
#include "node_internals.h"
#include "node_revert.h"
#include "node_version.h"
#include "tracing/traced_value.h"
@ -131,6 +130,7 @@ typedef int mode_t;
namespace node {
using native_module::NativeModule;
using options_parser::kAllowedInEnvironment;
using options_parser::kDisallowedInEnvironment;
using v8::Array;
@ -775,6 +775,7 @@ static MaybeLocal<Value> ExecuteString(Environment* env,
try_catch.SetVerbose(false);
ScriptOrigin origin(filename);
MaybeLocal<Script> script =
Script::Compile(env->context(), source, &origin);
if (script.IsEmpty()) {
@ -1222,19 +1223,7 @@ static void GetInternalBinding(const FunctionCallbackInfo<Value>& args) {
DefineConstants(env->isolate(), exports);
} else if (!strcmp(*module_v, "natives")) {
exports = Object::New(env->isolate());
DefineJavaScript(env, exports);
} else if (!strcmp(*module_v, "code_cache")) {
// internalBinding('code_cache')
exports = Object::New(env->isolate());
DefineCodeCache(env, exports);
} else if (!strcmp(*module_v, "code_cache_hash")) {
// internalBinding('code_cache_hash')
exports = Object::New(env->isolate());
DefineCodeCacheHash(env, exports);
} else if (!strcmp(*module_v, "natives_hash")) {
// internalBinding('natives_hash')
exports = Object::New(env->isolate());
DefineJavaScriptHash(env, exports);
NativeModule::GetNatives(env, exports);
} else {
return ThrowIfNoSuchModule(env, *module_v);
}
@ -1772,6 +1761,8 @@ void LoadEnvironment(Environment* env) {
// lib/internal/bootstrap/node.js, each included as a static C string
// defined in node_javascript.h, generated in node_javascript.cc by
// node_js2c.
// TODO(joyeecheung): use NativeModule::Compile
Local<String> loaders_name =
FIXED_ONE_BYTE_STRING(env->isolate(), "internal/bootstrap/loaders.js");
MaybeLocal<Function> loaders_bootstrapper =
@ -1831,6 +1822,8 @@ void LoadEnvironment(Environment* env) {
env->options()->debug_options->break_node_first_line)
};
NativeModule::LoadBindings(env);
// Bootstrap internal loaders
Local<Value> bootstrapped_loaders;
if (!ExecuteBootstrapper(env, loaders_bootstrapper.ToLocalChecked(),
@ -2484,6 +2477,8 @@ Local<Context> NewContext(Isolate* isolate,
{
// Run lib/internal/per_context.js
Context::Scope context_scope(context);
// TODO(joyeecheung): use NativeModule::Compile
Local<String> per_context = NodePerContextSource(isolate);
ScriptCompiler::Source per_context_src(per_context, nullptr);
Local<Script> s = ScriptCompiler::Compile(

2
src/node_code_cache.h
View File

@ -7,6 +7,8 @@
namespace node {
extern const bool native_module_has_code_cache;
void DefineCodeCache(Environment* env, v8::Local<v8::Object> target);
void DefineCodeCacheHash(Environment* env, v8::Local<v8::Object> target);

3
src/node_code_cache_stub.cc
View File

@ -5,6 +5,9 @@
// The stub here is used when configure is run without `--code-cache-path`
namespace node {
const bool native_module_has_code_cache = false;
void DefineCodeCache(Environment* env, v8::Local<v8::Object> target) {
// When we do not produce code cache for builtin modules,
// `internalBinding('code_cache')` returns an empty object

1
src/node_internals.h
View File

@ -118,6 +118,7 @@ struct sockaddr;
V(js_stream) \
V(messaging) \
V(module_wrap) \
V(native_module) \
V(options) \
V(os) \
V(performance) \

326
src/node_native_module.cc Normal file
View File

@ -0,0 +1,326 @@
#include "node_native_module.h"
#include "node_code_cache.h"
#include "node_errors.h"
#include "node_javascript.h"
namespace node {
namespace native_module {
using v8::Array;
using v8::ArrayBuffer;
using v8::Context;
using v8::EscapableHandleScope;
using v8::Function;
using v8::FunctionCallbackInfo;
using v8::HandleScope;
using v8::Integer;
using v8::IntegrityLevel;
using v8::Isolate;
using v8::Local;
using v8::Maybe;
using v8::MaybeLocal;
using v8::Object;
using v8::Script;
using v8::ScriptCompiler;
using v8::ScriptOrigin;
using v8::Set;
using v8::String;
using v8::TryCatch;
using v8::Uint8Array;
using v8::Value;
void NativeModule::GetNatives(Environment* env, Local<Object> exports) {
DefineJavaScript(env, exports);
}
void NativeModule::LoadBindings(Environment* env) {
// TODO(joyeecheung): put the static values into a
// std::map<std::string, const uint8_t*> instead of a v8::Object,
// because here they are only looked up from the C++ side
// (except in process.binding('natives') which we don't use)
// so there is little value to put them in a v8::Object upfront.
// Moreover, a std::map lookup should be faster than a lookup on
// an V8 Object in dictionary mode.
Isolate* isolate = env->isolate();
Local<Context> context = env->context();
Local<Value> null = Null(isolate);
Local<Object> native_modules_source = Object::New(isolate);
CHECK(native_modules_source->SetPrototype(context, null).FromJust());
DefineJavaScript(env, native_modules_source);
native_modules_source->SetIntegrityLevel(context, IntegrityLevel::kFrozen)
.FromJust();
env->set_native_modules_source(native_modules_source);
Local<Object> native_modules_source_hash = Object::New(isolate);
CHECK(native_modules_source_hash->SetPrototype(context, null).FromJust());
DefineJavaScriptHash(env, native_modules_source_hash);
native_modules_source_hash
->SetIntegrityLevel(context, IntegrityLevel::kFrozen)
.FromJust();
env->set_native_modules_source_hash(native_modules_source_hash);
Local<Object> native_modules_code_cache = Object::New(isolate);
CHECK(native_modules_code_cache->SetPrototype(context, null).FromJust());
DefineCodeCache(env, native_modules_code_cache);
native_modules_code_cache->SetIntegrityLevel(context, IntegrityLevel::kFrozen)
.FromJust();
env->set_native_modules_code_cache(native_modules_code_cache);
Local<Object> native_modules_code_cache_hash = Object::New(isolate);
CHECK(native_modules_code_cache_hash->SetPrototype(context, null).FromJust());
DefineCodeCacheHash(env, native_modules_code_cache_hash);
native_modules_code_cache_hash
->SetIntegrityLevel(context, IntegrityLevel::kFrozen)
.FromJust();
env->set_native_modules_code_cache_hash(native_modules_code_cache_hash);
env->set_native_modules_with_cache(Set::New(isolate));
env->set_native_modules_without_cache(Set::New(isolate));
}
void NativeModule::CompileCodeCache(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
CHECK(args[0]->IsString());
Local<String> id = args[0].As<String>();
Local<Value> result = CompileAsModule(env, id, true);
if (!result.IsEmpty()) {
args.GetReturnValue().Set(result);
}
}
void NativeModule::CompileFunction(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
CHECK(args[0]->IsString());
Local<String> id = args[0].As<String>();
Local<Value> result = CompileAsModule(env, id, false);
if (!result.IsEmpty()) {
args.GetReturnValue().Set(result);
}
}
Local<Value> NativeModule::CompileAsModule(Environment* env,
Local<String> id,
bool produce_code_cache) {
Local<String> parameters[] = {env->exports_string(),
env->require_string(),
env->module_string(),
env->process_string(),
env->internal_binding_string()};
return Compile(
env, id, parameters, arraysize(parameters), produce_code_cache);
}
// Currently V8 only checks that the length of the source code is the
// same as the code used to generate the hash, so we add an additional
// check here:
// 1. During compile time, when generating node_javascript.cc and
// node_code_cache.cc, we compute and include the hash of the
// JavaScript source in both.
// 2. At runtime, we check that the hash of the code being compiled
// and the hash of the code used to generate the cache
// (without the parameters) is the same.
// This is based on the assumptions:
// 1. `code_cache_hash` must be in sync with `code_cache`
// (both defined in node_code_cache.cc)
// 2. `source_hash` must be in sync with `source`
// (both defined in node_javascript.cc)
// 3. If `source_hash` is in sync with `code_cache_hash`,
// then the source code used to generate `code_cache`
// should be in sync with the source code in `source`
// The only variable left, then, are the parameters passed to the
// CompileFunctionInContext. If the parameters used generate the cache
// is different from the one used to compile modules at run time, then
// there could be false postivies, but that should be rare and should fail
// early in the bootstrap process so it should be easy to detect and fix.
// Returns nullptr if there is no code cache corresponding to the id
ScriptCompiler::CachedData* GetCachedData(Environment* env, Local<String> id) {
HandleScope scope(env->isolate());
Local<Context> context = env->context();
Local<Value> result =
env->native_modules_code_cache()->Get(context, id).ToLocalChecked();
// This could be false if the module cannot be cached somehow.
// See lib/internal/bootstrap/cache.js on the modules that cannot be cached
if (result->IsUndefined()) {
return nullptr;
}
CHECK(result->IsUint8Array());
Local<Uint8Array> code_cache = result.As<Uint8Array>();
result =
env->native_modules_code_cache_hash()->Get(context, id).ToLocalChecked();
CHECK(result->IsString());
Local<String> code_cache_hash = result.As<String>();
result =
env->native_modules_source_hash()->Get(context, id).ToLocalChecked();
CHECK(result->IsString());
Local<String> source_hash = result.As<String>();
// It may fail when any of the inputs of the `node_js2c` target in
// node.gyp is modified but the tools/generate_code_cache.js
// is not re run.
// FIXME(joyeecheung): Figure out how to resolve the dependency issue.
// When the code cache was introduced we were at a point where refactoring
// node.gyp may not be worth the effort.
CHECK(code_cache_hash->StrictEquals(source_hash));
ArrayBuffer::Contents contents = code_cache->Buffer()->GetContents();
uint8_t* data = static_cast<uint8_t*>(contents.Data());
return new ScriptCompiler::CachedData(data + code_cache->ByteOffset(),
code_cache->ByteLength());
}
// Returns Local<Function> of the compiled module if produce_code_cache
// is false (we are only compiling the function).
// Otherwise return a Local<Object> containing the cache.
Local<Value> NativeModule::Compile(Environment* env,
Local<String> id,
Local<String> parameters[],
size_t parameters_count,
bool produce_code_cache) {
EscapableHandleScope scope(env->isolate());
Local<Context> context = env->context();
Isolate* isolate = env->isolate();
Local<Value> result =
env->native_modules_source()->Get(context, id).ToLocalChecked();
CHECK(result->IsString());
Local<String> source = result.As<String>();
Local<String> filename =
String::Concat(isolate, id, FIXED_ONE_BYTE_STRING(isolate, ".js"));
Local<Integer> line_offset = Integer::New(isolate, 0);
Local<Integer> column_offset = Integer::New(isolate, 0);
ScriptOrigin origin(filename, line_offset, column_offset);
bool use_cache = false;
ScriptCompiler::CachedData* cached_data = nullptr;
// 1. We won't even check the existence of the cache if the binary is not
// built with them.
// 2. If we are generating code cache for tools/general_code_cache.js, we
// are not going to use any cache ourselves.
if (native_module_has_code_cache && !produce_code_cache) {
cached_data = GetCachedData(env, id);
if (cached_data != nullptr) {
use_cache = true;
}
}
ScriptCompiler::Source script_source(source, origin, cached_data);
ScriptCompiler::CompileOptions options;
if (produce_code_cache) {
options = ScriptCompiler::kEagerCompile;
} else if (use_cache) {
options = ScriptCompiler::kConsumeCodeCache;
} else {
options = ScriptCompiler::kNoCompileOptions;
}
MaybeLocal<Function> maybe_fun =
ScriptCompiler::CompileFunctionInContext(context,
&script_source,
parameters_count,
parameters,
0,
nullptr,
options);
TryCatch try_catch(isolate);
Local<Function> fun;
// This could fail when there are early errors in the native modules,
// e.g. the syntax errors
if (maybe_fun.IsEmpty() || !maybe_fun.ToLocal(&fun)) {
DecorateErrorStack(env, try_catch);
try_catch.ReThrow();
return scope.Escape(Local<Value>());
}
if (use_cache) {
// If the cache is rejected, something must be wrong with the build
// and we should just crash.
CHECK(!script_source.GetCachedData()->rejected);
if (env->native_modules_with_cache()->Add(context, id).IsEmpty()) {
return scope.Escape(Local<Value>());
}
} else {
if (env->native_modules_without_cache()->Add(context, id).IsEmpty()) {
return scope.Escape(Local<Value>());
}
}
if (produce_code_cache) {
std::unique_ptr<ScriptCompiler::CachedData> cached_data(
ScriptCompiler::CreateCodeCacheForFunction(fun));
CHECK_NE(cached_data, nullptr);
char* data =
reinterpret_cast<char*>(const_cast<uint8_t*>(cached_data->data));
// Since we have no API to create a buffer from a new'ed pointer,
// we will need to copy it - but this code path is only run by the
// tooling that generates the code cache to be bundled in the binary
// so it should be fine.
Local<Object> buf =
Buffer::Copy(env, data, cached_data->length).ToLocalChecked();
return scope.Escape(buf);
} else {
return scope.Escape(fun);
}
}
void Initialize(Local<Object> target,
Local<Value> unused,
Local<Context> context) {
Environment* env = Environment::GetCurrent(context);
target
->Set(context,
FIXED_ONE_BYTE_STRING(env->isolate(), "source"),
env->native_modules_source())
.FromJust();
target
->Set(context,
FIXED_ONE_BYTE_STRING(env->isolate(), "sourceHash"),
env->native_modules_source_hash())
.FromJust();
target
->Set(context,
FIXED_ONE_BYTE_STRING(env->isolate(), "codeCache"),
env->native_modules_code_cache())
.FromJust();
target
->Set(context,
FIXED_ONE_BYTE_STRING(env->isolate(), "codeCacheHash"),
env->native_modules_code_cache_hash())
.FromJust();
target
->Set(context,
FIXED_ONE_BYTE_STRING(env->isolate(), "compiledWithCache"),
env->native_modules_with_cache())
.FromJust();
target
->Set(context,
FIXED_ONE_BYTE_STRING(env->isolate(), "compiledWithoutCache"),
env->native_modules_without_cache())
.FromJust();
env->SetMethod(target, "compileFunction", NativeModule::CompileFunction);
env->SetMethod(target, "compileCodeCache", NativeModule::CompileCodeCache);
// internalBinding('native_module') should be frozen
target->SetIntegrityLevel(context, IntegrityLevel::kFrozen).FromJust();
}
} // namespace native_module
} // namespace node
NODE_MODULE_CONTEXT_AWARE_INTERNAL(native_module,
node::native_module::Initialize)

41
src/node_native_module.h Normal file
View File

@ -0,0 +1,41 @@
#ifndef SRC_NODE_NATIVE_MODULE_H_
#define SRC_NODE_NATIVE_MODULE_H_
#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS
#include "node_internals.h"
namespace node {
namespace native_module {
// The native (C++) side of the native module compilation.
class NativeModule {
public:
// For legacy process.binding('natives') which is mutable
static void GetNatives(Environment* env, v8::Local<v8::Object> exports);
// Loads the static JavaScript source code and the cache into Environment
static void LoadBindings(Environment* env);
// Compile code cache for a specific native module
static void CompileCodeCache(const v8::FunctionCallbackInfo<v8::Value>& args);
// Compile a specific native module as a function
static void CompileFunction(const v8::FunctionCallbackInfo<v8::Value>& args);
private:
static v8::Local<v8::Value> CompileAsModule(Environment* env,
v8::Local<v8::String> id,
bool produce_code_cache);
// TODO(joyeecheung): make this public and reuse it to compile bootstrappers
static v8::Local<v8::Value> Compile(Environment* env,
v8::Local<v8::String> id,
v8::Local<v8::String> parameters[],
size_t parameters_count,
bool produce_code_cache);
};
} // namespace native_module
} // namespace node
#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS
#endif // SRC_NODE_NATIVE_MODULE_H_

55
test/code-cache/test-code-cache.js
View File

@ -7,17 +7,28 @@
require('../common');
const assert = require('assert');
const {
types: {
isUint8Array
}
} = require('util');
const {
cachableBuiltins,
codeCache,
compiledWithCache,
compiledWithoutCache
cannotUseCache
} = require('internal/bootstrap/cache');
const {
isMainThread
} = require('worker_threads');
const {
internalBinding
} = require('internal/test/binding');
const {
compiledWithoutCache,
compiledWithCache
} = internalBinding('native_module');
for (const key of cachableBuiltins) {
if (!isMainThread && key === 'trace_events') {
continue; // Cannot load trace_events in workers
}
require(key);
}
const loadedModules = process.moduleLoadList
.filter((m) => m.startsWith('NativeModule'))
@ -26,29 +37,23 @@ const loadedModules = process.moduleLoadList
// The binary is not configured with code cache, verifies that the builtins
// are all compiled without cache and we are doing the bookkeeping right.
if (process.config.variables.node_code_cache_path === undefined) {
assert.deepStrictEqual(compiledWithCache, []);
assert.notStrictEqual(compiledWithoutCache.length, 0);
for (const key of loadedModules) {
assert(compiledWithoutCache.includes(key),
`"${key}" should not have been compiled with code cache`);
}
console.log('The binary is not configured with code cache');
assert.deepStrictEqual(compiledWithCache, new Set());
assert.deepStrictEqual(compiledWithoutCache, new Set(loadedModules));
} else {
// The binary is configured with code cache.
console.log('The binary is configured with code cache');
assert.strictEqual(
typeof process.config.variables.node_code_cache_path,
'string'
);
assert.deepStrictEqual(compiledWithoutCache, []);
for (const key of loadedModules) {
assert(compiledWithCache.includes(key),
`"${key}" should've been compiled with code cache`);
}
for (const key of cachableBuiltins) {
assert(isUint8Array(codeCache[key]) && codeCache[key].length > 0,
`Code cache for "${key}" should've been generated`);
if (cannotUseCache.includes(key)) {
assert(compiledWithoutCache.has(key),
`"${key}" should've been compiled without code cache`);
} else {
assert(compiledWithCache.has(key),
`"${key}" should've been compiled with code cache`);
}
}
}

4
tools/generate_code_cache.js
View File

@ -8,8 +8,8 @@
// of `configure`.
const {
getCodeCache,
getSource,
getCodeCache,
cachableBuiltins
} = require('internal/bootstrap/cache');
@ -118,6 +118,8 @@ namespace node {
${cacheDefinitions.join('\n\n')}
const bool native_module_has_code_cache = true;
// The target here will be returned as \`internalBinding('code_cache')\`
void DefineCodeCache(Environment* env, v8::Local<v8::Object> target) {
v8::Isolate* isolate = env->isolate();