module: support main w/o extension, pjson cache

This adds support for ensuring that the top-level main into Node is
supported loading when it has no extension for backwards-compat with
NodeJS bin workflows.

In addition package.json caching is implemented in the module lookup
process.

PR-URL: https://github.com/nodejs/node/pull/18728
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
This commit is contained in:
Guy Bedford 2018-02-12 13:02:42 +02:00 committed by guybedford
parent 0e7b61229a
commit f1fc426cce
13 changed files with 210 additions and 139 deletions

View File

@ -117,9 +117,12 @@ The resolve hook returns the resolved file URL and module format for a
given module specifier and parent file URL:
```js
import url from 'url';
const baseURL = new URL('file://');
baseURL.pathname = process.cwd() + '/';
export async function resolve(specifier, parentModuleURL, defaultResolver) {
export async function resolve(specifier,
parentModuleURL = baseURL,
defaultResolver) {
return {
url: new URL(specifier, parentModuleURL).href,
format: 'esm'
@ -127,7 +130,9 @@ export async function resolve(specifier, parentModuleURL, defaultResolver) {
}
```
The default NodeJS ES module resolution function is provided as a third
The parentURL is provided as `undefined` when performing main Node.js load itself.
The default Node.js ES module resolution function is provided as a third
argument to the resolver for easy compatibility workflows.
In addition to returning the resolved file URL value, the resolve hook also
@ -155,7 +160,10 @@ import Module from 'module';
const builtins = Module.builtinModules;
const JS_EXTENSIONS = new Set(['.js', '.mjs']);
export function resolve(specifier, parentModuleURL/*, defaultResolve */) {
const baseURL = new URL('file://');
baseURL.pathname = process.cwd() + '/';
export function resolve(specifier, parentModuleURL = baseURL, defaultResolve) {
if (builtins.includes(specifier)) {
return {
url: specifier,

View File

@ -2,7 +2,6 @@
const { URL } = require('url');
const CJSmodule = require('module');
const internalURLModule = require('internal/url');
const internalFS = require('internal/fs');
const NativeModule = require('native_module');
const { extname } = require('path');
@ -11,6 +10,7 @@ const preserveSymlinks = !!process.binding('config').preserveSymlinks;
const errors = require('internal/errors');
const { resolve: moduleWrapResolve } = internalBinding('module_wrap');
const StringStartsWith = Function.call.bind(String.prototype.startsWith);
const { getURLFromFilePath, getPathFromURL } = require('internal/url');
const realpathCache = new Map();
@ -57,7 +57,8 @@ function resolve(specifier, parentURL) {
let url;
try {
url = search(specifier, parentURL);
url = search(specifier,
parentURL || getURLFromFilePath(`${process.cwd()}/`).href);
} catch (e) {
if (typeof e.message === 'string' &&
StringStartsWith(e.message, 'Cannot find module'))
@ -66,17 +67,27 @@ function resolve(specifier, parentURL) {
}
if (!preserveSymlinks) {
const real = realpathSync(internalURLModule.getPathFromURL(url), {
const real = realpathSync(getPathFromURL(url), {
[internalFS.realpathCacheKey]: realpathCache
});
const old = url;
url = internalURLModule.getURLFromFilePath(real);
url = getURLFromFilePath(real);
url.search = old.search;
url.hash = old.hash;
}
const ext = extname(url.pathname);
return { url: `${url}`, format: extensionFormatMap[ext] || ext };
let format = extensionFormatMap[ext];
if (!format) {
const isMain = parentURL === undefined;
if (isMain)
format = 'cjs';
else
throw new errors.Error('ERR_UNKNOWN_FILE_EXTENSION', url.pathname);
}
return { url: `${url}`, format };
}
module.exports = resolve;

View File

@ -1,51 +1,21 @@
'use strict';
const path = require('path');
const { getURLFromFilePath, URL } = require('internal/url');
const errors = require('internal/errors');
const ModuleMap = require('internal/loader/ModuleMap');
const ModuleJob = require('internal/loader/ModuleJob');
const defaultResolve = require('internal/loader/DefaultResolve');
const createDynamicModule = require('internal/loader/CreateDynamicModule');
const translators = require('internal/loader/Translators');
const { setImportModuleDynamicallyCallback } = internalBinding('module_wrap');
const FunctionBind = Function.call.bind(Function.prototype.bind);
const debug = require('util').debuglog('esm');
// Returns a file URL for the current working directory.
function getURLStringForCwd() {
try {
return getURLFromFilePath(`${process.cwd()}/`).href;
} catch (e) {
e.stack;
// If the current working directory no longer exists.
if (e.code === 'ENOENT') {
return undefined;
}
throw e;
}
}
function normalizeReferrerURL(referrer) {
if (typeof referrer === 'string' && path.isAbsolute(referrer)) {
return getURLFromFilePath(referrer).href;
}
return new URL(referrer).href;
}
/* A Loader instance is used as the main entry point for loading ES modules.
* Currently, this is a singleton -- there is only one used for loading
* the main module and everything in its dependency graph. */
class Loader {
constructor(base = getURLStringForCwd()) {
if (typeof base !== 'string')
throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'string');
this.base = base;
this.isMain = true;
constructor() {
// methods which translate input code or other information
// into es modules
this.translators = translators;
@ -71,8 +41,9 @@ class Loader {
this._dynamicInstantiate = undefined;
}
async resolve(specifier, parentURL = this.base) {
if (typeof parentURL !== 'string')
async resolve(specifier, parentURL) {
const isMain = parentURL === undefined;
if (!isMain && typeof parentURL !== 'string')
throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'parentURL', 'string');
const { url, format } =
@ -93,7 +64,7 @@ class Loader {
return { url, format };
}
async import(specifier, parent = this.base) {
async import(specifier, parent) {
const job = await this.getModuleJob(specifier, parent);
const module = await job.run();
return module.namespace();
@ -107,7 +78,7 @@ class Loader {
this._dynamicInstantiate = FunctionBind(dynamicInstantiate, null);
}
async getModuleJob(specifier, parentURL = this.base) {
async getModuleJob(specifier, parentURL) {
const { url, format } = await this.resolve(specifier, parentURL);
let job = this.moduleMap.get(url);
if (job !== undefined)
@ -134,24 +105,16 @@ class Loader {
}
let inspectBrk = false;
if (this.isMain) {
if (process._breakFirstLine) {
delete process._breakFirstLine;
inspectBrk = true;
}
this.isMain = false;
if (process._breakFirstLine) {
delete process._breakFirstLine;
inspectBrk = true;
}
job = new ModuleJob(this, url, loaderInstance, inspectBrk);
this.moduleMap.set(url, job);
return job;
}
static registerImportDynamicallyCallback(loader) {
setImportModuleDynamicallyCallback(async (referrer, specifier) => {
return loader.import(specifier, normalizeReferrerURL(referrer));
});
}
}
Object.setPrototypeOf(Loader.prototype, null);
module.exports = Loader;

View File

@ -19,7 +19,7 @@ const JsonParse = JSON.parse;
const translators = new SafeMap();
module.exports = translators;
// Stragety for loading a standard JavaScript module
// Strategy for loading a standard JavaScript module
translators.set('esm', async (url) => {
const source = `${await readFileAsync(new URL(url))}`;
debug(`Translating StandardModule ${url}`);
@ -62,7 +62,7 @@ translators.set('builtin', async (url) => {
});
});
// Stragety for loading a node native module
// Strategy for loading a node native module
translators.set('addon', async (url) => {
debug(`Translating NativeModule ${url}`);
return createDynamicModule(['default'], url, (reflect) => {
@ -74,7 +74,7 @@ translators.set('addon', async (url) => {
});
});
// Stragety for loading a JSON file
// Strategy for loading a JSON file
translators.set('json', async (url) => {
debug(`Translating JSONModule ${url}`);
return createDynamicModule(['default'], url, (reflect) => {

View File

@ -1,17 +1,54 @@
'use strict';
const {
setImportModuleDynamicallyCallback,
setInitializeImportMetaObjectCallback
} = internalBinding('module_wrap');
const { getURLFromFilePath } = require('internal/url');
const Loader = require('internal/loader/Loader');
const path = require('path');
const { URL } = require('url');
function normalizeReferrerURL(referrer) {
if (typeof referrer === 'string' && path.isAbsolute(referrer)) {
return getURLFromFilePath(referrer).href;
}
return new URL(referrer).href;
}
function initializeImportMetaObject(wrap, meta) {
meta.url = wrap.url;
}
function setupModules() {
setInitializeImportMetaObjectCallback(initializeImportMetaObject);
}
let loaderResolve;
exports.loaderPromise = new Promise((resolve, reject) => {
loaderResolve = resolve;
});
module.exports = {
setup: setupModules
exports.ESMLoader = undefined;
exports.setup = function() {
setInitializeImportMetaObjectCallback(initializeImportMetaObject);
let ESMLoader = new Loader();
const loaderPromise = (async () => {
const userLoader = process.binding('config').userLoader;
if (userLoader) {
const hooks = await ESMLoader.import(
userLoader, getURLFromFilePath(`${process.cwd()}/`).href);
ESMLoader = new Loader();
ESMLoader.hook(hooks);
exports.ESMLoader = ESMLoader;
}
return ESMLoader;
})();
loaderResolve(loaderPromise);
setImportModuleDynamicallyCallback(async (referrer, specifier) => {
const loader = await loaderPromise;
return loader.import(specifier, normalizeReferrerURL(referrer));
});
exports.ESMLoader = ESMLoader;
};

View File

@ -24,7 +24,6 @@
const NativeModule = require('native_module');
const util = require('util');
const { decorateErrorStack } = require('internal/util');
const internalModule = require('internal/module');
const { getURLFromFilePath } = require('internal/url');
const vm = require('vm');
const assert = require('assert').ok;
@ -35,6 +34,7 @@ const {
internalModuleReadJSON,
internalModuleStat
} = process.binding('fs');
const internalModule = require('internal/module');
const preserveSymlinks = !!process.binding('config').preserveSymlinks;
const experimentalModules = !!process.binding('config').experimentalModules;
@ -43,10 +43,9 @@ const errors = require('internal/errors');
module.exports = Module;
// these are below module.exports for the circular reference
const Loader = require('internal/loader/Loader');
const internalESModule = require('internal/process/modules');
const ModuleJob = require('internal/loader/ModuleJob');
const createDynamicModule = require('internal/loader/CreateDynamicModule');
let ESMLoader;
function stat(filename) {
filename = path.toNamespacedPath(filename);
@ -447,7 +446,6 @@ Module._resolveLookupPaths = function(request, parent, newReturn) {
return (newReturn ? parentDir : [id, parentDir]);
};
// Check the cache for the requested file.
// 1. If a module already exists in the cache: return its exports object.
// 2. If the module is native: call `NativeModule.require()` with the
@ -460,22 +458,10 @@ Module._load = function(request, parent, isMain) {
debug('Module._load REQUEST %s parent: %s', request, parent.id);
}
if (isMain && experimentalModules) {
(async () => {
// loader setup
if (!ESMLoader) {
ESMLoader = new Loader();
const userLoader = process.binding('config').userLoader;
if (userLoader) {
ESMLoader.isMain = false;
const hooks = await ESMLoader.import(userLoader);
ESMLoader = new Loader();
ESMLoader.hook(hooks);
}
}
Loader.registerImportDynamicallyCallback(ESMLoader);
await ESMLoader.import(getURLFromFilePath(request).pathname);
})()
if (experimentalModules && isMain) {
internalESModule.loaderPromise.then((loader) => {
return loader.import(getURLFromFilePath(request).pathname);
})
.catch((e) => {
decorateErrorStack(e);
console.error(e);
@ -578,7 +564,8 @@ Module.prototype.load = function(filename) {
Module._extensions[extension](this, filename);
this.loaded = true;
if (ESMLoader) {
if (experimentalModules) {
const ESMLoader = internalESModule.ESMLoader;
const url = getURLFromFilePath(filename);
const urlString = `${url}`;
const exports = this.exports;

View File

@ -54,7 +54,26 @@ class performance_state;
namespace loader {
class ModuleWrap;
}
struct Exists {
enum Bool { Yes, No };
};
struct IsValid {
enum Bool { Yes, No };
};
struct HasMain {
enum Bool { Yes, No };
};
struct PackageConfig {
const Exists::Bool exists;
const IsValid::Bool is_valid;
const HasMain::Bool has_main;
const std::string main;
};
} // namespace loader
// Pick an index that's hopefully out of the way when we're embedded inside
// another application. Performance-wise or memory-wise it doesn't matter:
@ -609,6 +628,8 @@ class Environment {
std::unordered_multimap<int, loader::ModuleWrap*> module_map;
std::unordered_map<std::string, loader::PackageConfig> package_json_cache;
inline double* heap_statistics_buffer() const;
inline void set_heap_statistics_buffer(double* pointer);

View File

@ -461,10 +461,9 @@ enum CheckFileOptions {
CLOSE_AFTER_CHECK
};
Maybe<uv_file> CheckFile(const URL& search,
Maybe<uv_file> CheckFile(const std::string& path,
CheckFileOptions opt = CLOSE_AFTER_CHECK) {
uv_fs_t fs_req;
std::string path = search.ToFilePath();
if (path.empty()) {
return Nothing<uv_file>();
}
@ -481,19 +480,74 @@ Maybe<uv_file> CheckFile(const URL& search,
uv_fs_req_cleanup(&fs_req);
if (is_directory) {
uv_fs_close(nullptr, &fs_req, fd, nullptr);
CHECK_EQ(0, uv_fs_close(nullptr, &fs_req, fd, nullptr));
uv_fs_req_cleanup(&fs_req);
return Nothing<uv_file>();
}
if (opt == CLOSE_AFTER_CHECK) {
uv_fs_close(nullptr, &fs_req, fd, nullptr);
CHECK_EQ(0, uv_fs_close(nullptr, &fs_req, fd, nullptr));
uv_fs_req_cleanup(&fs_req);
}
return Just(fd);
}
const PackageConfig& GetPackageConfig(Environment* env,
const std::string path) {
auto existing = env->package_json_cache.find(path);
if (existing != env->package_json_cache.end()) {
return existing->second;
}
Maybe<uv_file> check = CheckFile(path, LEAVE_OPEN_AFTER_CHECK);
if (check.IsNothing()) {
auto entry = env->package_json_cache.emplace(path,
PackageConfig { Exists::No, IsValid::Yes, HasMain::No, "" });
return entry.first->second;
}
Isolate* isolate = env->isolate();
v8::HandleScope handle_scope(isolate);
std::string pkg_src = ReadFile(check.FromJust());
uv_fs_t fs_req;
CHECK_EQ(0, uv_fs_close(nullptr, &fs_req, check.FromJust(), nullptr));
uv_fs_req_cleanup(&fs_req);
Local<String> src;
if (!String::NewFromUtf8(isolate,
pkg_src.c_str(),
v8::NewStringType::kNormal,
pkg_src.length()).ToLocal(&src)) {
auto entry = env->package_json_cache.emplace(path,
PackageConfig { Exists::No, IsValid::Yes, HasMain::No, "" });
return entry.first->second;
}
Local<Value> pkg_json_v;
Local<Object> pkg_json;
if (!JSON::Parse(env->context(), src).ToLocal(&pkg_json_v) ||
!pkg_json_v->ToObject(env->context()).ToLocal(&pkg_json)) {
auto entry = env->package_json_cache.emplace(path,
PackageConfig { Exists::Yes, IsValid::No, HasMain::No, "" });
return entry.first->second;
}
Local<Value> pkg_main;
HasMain::Bool has_main = HasMain::No;
std::string main_std;
if (pkg_json->Get(env->context(), env->main_string()).ToLocal(&pkg_main)) {
has_main = HasMain::Yes;
Utf8Value main_utf8(isolate, pkg_main);
main_std.assign(std::string(*main_utf8, main_utf8.length()));
}
auto entry = env->package_json_cache.emplace(path,
PackageConfig { Exists::Yes, IsValid::Yes, has_main, "" });
return entry.first->second;
}
enum ResolveExtensionsOptions {
TRY_EXACT_NAME,
ONLY_VIA_EXTENSIONS
@ -502,7 +556,8 @@ enum ResolveExtensionsOptions {
template<ResolveExtensionsOptions options>
Maybe<URL> ResolveExtensions(const URL& search) {
if (options == TRY_EXACT_NAME) {
Maybe<uv_file> check = CheckFile(search);
std::string filePath = search.ToFilePath();
Maybe<uv_file> check = CheckFile(filePath);
if (!check.IsNothing()) {
return Just(search);
}
@ -510,7 +565,7 @@ Maybe<URL> ResolveExtensions(const URL& search) {
for (const char* extension : EXTENSIONS) {
URL guess(search.path() + extension, &search);
Maybe<uv_file> check = CheckFile(guess);
Maybe<uv_file> check = CheckFile(guess.ToFilePath());
if (!check.IsNothing()) {
return Just(guess);
}
@ -525,44 +580,18 @@ inline Maybe<URL> ResolveIndex(const URL& search) {
Maybe<URL> ResolveMain(Environment* env, const URL& search) {
URL pkg("package.json", &search);
Maybe<uv_file> check = CheckFile(pkg, LEAVE_OPEN_AFTER_CHECK);
if (check.IsNothing()) {
const PackageConfig& pjson =
GetPackageConfig(env, pkg.ToFilePath());
// Note invalid package.json should throw in resolver
// currently we silently ignore which is incorrect
if (!pjson.exists || !pjson.is_valid || !pjson.has_main) {
return Nothing<URL>();
}
Isolate* isolate = env->isolate();
Local<Context> context = isolate->GetCurrentContext();
std::string pkg_src = ReadFile(check.FromJust());
uv_fs_t fs_req;
uv_fs_close(nullptr, &fs_req, check.FromJust(), nullptr);
uv_fs_req_cleanup(&fs_req);
// It's not okay for the called of this method to not be able to tell
// whether an exception is pending or not.
TryCatch try_catch(isolate);
Local<String> src;
if (!String::NewFromUtf8(isolate,
pkg_src.c_str(),
v8::NewStringType::kNormal,
pkg_src.length()).ToLocal(&src)) {
return Nothing<URL>();
if (!ShouldBeTreatedAsRelativeOrAbsolutePath(pjson.main)) {
return Resolve(env, "./" + pjson.main, search);
}
Local<Value> pkg_json;
if (!JSON::Parse(context, src).ToLocal(&pkg_json) || !pkg_json->IsObject())
return Nothing<URL>();
Local<Value> pkg_main;
if (!pkg_json.As<Object>()->Get(context, env->main_string())
.ToLocal(&pkg_main) || !pkg_main->IsString()) {
return Nothing<URL>();
}
Utf8Value main_utf8(isolate, pkg_main.As<String>());
std::string main_std(*main_utf8, main_utf8.length());
if (!ShouldBeTreatedAsRelativeOrAbsolutePath(main_std)) {
main_std.insert(0, "./");
}
return Resolve(env, main_std, search);
return Resolve(env, pjson.main, search);
}
Maybe<URL> ResolveModule(Environment* env,
@ -572,7 +601,8 @@ Maybe<URL> ResolveModule(Environment* env,
URL dir("");
do {
dir = parent;
Maybe<URL> check = Resolve(env, "./node_modules/" + specifier, dir, true);
Maybe<URL> check =
Resolve(env, "./node_modules/" + specifier, dir, IgnoreMain);
if (!check.IsNothing()) {
const size_t limit = specifier.find('/');
const size_t spec_len =
@ -594,8 +624,8 @@ Maybe<URL> ResolveModule(Environment* env,
Maybe<URL> ResolveDirectory(Environment* env,
const URL& search,
bool read_pkg_json) {
if (read_pkg_json) {
PackageMainCheck check_pjson_main) {
if (check_pjson_main) {
Maybe<URL> main = ResolveMain(env, search);
if (!main.IsNothing())
return main;
@ -605,15 +635,14 @@ Maybe<URL> ResolveDirectory(Environment* env,
} // anonymous namespace
Maybe<URL> Resolve(Environment* env,
const std::string& specifier,
const URL& base,
bool read_pkg_json) {
PackageMainCheck check_pjson_main) {
URL pure_url(specifier);
if (!(pure_url.flags() & URL_FLAGS_FAILED)) {
// just check existence, without altering
Maybe<uv_file> check = CheckFile(pure_url);
Maybe<uv_file> check = CheckFile(pure_url.ToFilePath());
if (check.IsNothing()) {
return Nothing<URL>();
}
@ -630,7 +659,7 @@ Maybe<URL> Resolve(Environment* env,
if (specifier.back() != '/') {
resolved = URL(specifier + "/", base);
}
return ResolveDirectory(env, resolved, read_pkg_json);
return ResolveDirectory(env, resolved, check_pjson_main);
} else {
return ResolveModule(env, specifier, base);
}
@ -667,7 +696,7 @@ void ModuleWrap::Resolve(const FunctionCallbackInfo<Value>& args) {
return;
}
Maybe<URL> result = node::loader::Resolve(env, specifier_std, url, true);
Maybe<URL> result = node::loader::Resolve(env, specifier_std, url);
if (result.IsNothing() || (result.FromJust().flags() & URL_FLAGS_FAILED)) {
std::string msg = "Cannot find module " + specifier_std;
env->ThrowError(msg.c_str());

View File

@ -12,10 +12,15 @@
namespace node {
namespace loader {
enum PackageMainCheck : bool {
CheckMain = true,
IgnoreMain = false
};
v8::Maybe<url::URL> Resolve(Environment* env,
const std::string& specifier,
const url::URL& base,
bool read_pkg_json = false);
PackageMainCheck read_pkg_json = CheckMain);
class ModuleWrap : public BaseObject {
public:

View File

@ -8,7 +8,10 @@ const builtins = new Set(
);
const JS_EXTENSIONS = new Set(['.js', '.mjs']);
export function resolve(specifier, parentModuleURL/*, defaultResolve */) {
const baseURL = new url.URL('file://');
baseURL.pathname = process.cwd() + '/';
export function resolve(specifier, parentModuleURL = baseURL /*, defaultResolve */) {
if (builtins.has(specifier)) {
return {
url: specifier,

View File

@ -3,7 +3,11 @@ const builtins = new Set(
Object.keys(process.binding('natives')).filter(str =>
/^(?!(?:internal|node|v8)\/)/.test(str))
)
export function resolve (specifier, base) {
const baseURL = new _url.URL('file://');
baseURL.pathname = process.cwd() + '/';
export function resolve (specifier, base = baseURL) {
if (builtins.has(specifier)) {
return {
url: specifier,

1
test/fixtures/es-modules/noext vendored Normal file
View File

@ -0,0 +1 @@
exports.cjs = true;

View File

@ -5,3 +5,5 @@ const { execFileSync } = require('child_process');
const node = process.argv[0];
execFileSync(node, ['--experimental-modules', 'test/es-module/test-esm-ok']);
execFileSync(node, ['--experimental-modules',
'test/fixtures/es-modules/noext']);