From d21a11dc23d6104b1d03fa2ddc1c808dcaf89c31 Mon Sep 17 00:00:00 2001 From: guybedford Date: Sun, 3 Sep 2017 13:20:06 +0200 Subject: [PATCH] module: resolve and instantiate loader pipeline hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This enables a --loader flag for Node, which can provide custom "resolve" and "dynamicInstantiate" methods for custom ES module loading. In the process, module providers have been converted from classes into functions and the module APIs have been made to pass URL strings over objects. PR-URL: https://github.com/nodejs/node/pull/15445 Reviewed-By: Bradley Farias Reviewed-By: James M Snell Reviewed-By: Jeremiah Senkpiel Reviewed-By: Michaël Zasso Reviewed-By: Timothy Gu --- .eslintrc.yaml | 3 +- doc/api/esm.md | 107 +++++++++++++++ lib/internal/errors.js | 2 + lib/internal/loader/Loader.js | 93 ++++++++----- lib/internal/loader/ModuleJob.js | 59 ++++----- lib/internal/loader/ModuleRequest.js | 122 ++++++++++++++++++ lib/internal/loader/resolveRequestUrl.js | 104 --------------- lib/internal/loader/search.js | 2 - lib/module.js | 48 +++---- node.gyp | 2 +- src/node.cc | 23 +++- src/node_config.cc | 12 +- src/node_internals.h | 5 + test/es-module/json.json | 3 + test/es-module/test-esm-addon.mjs | 7 + test/es-module/test-esm-example-loader.js | 6 + test/es-module/test-esm-json.mjs | 8 ++ test/es-module/test-esm-named-exports.mjs | 8 ++ test/es-module/test-esm-resolve-hook.mjs | 8 ++ .../builtin-named-exports-loader.mjs | 29 +++++ .../es-module-loaders/example-loader.mjs | 34 +++++ test/fixtures/es-module-loaders/js-as-esm.js | 1 + test/fixtures/es-module-loaders/js-loader.mjs | 19 +++ 23 files changed, 507 insertions(+), 198 deletions(-) create mode 100644 lib/internal/loader/ModuleRequest.js delete mode 100644 lib/internal/loader/resolveRequestUrl.js create mode 100644 test/es-module/json.json create mode 100644 test/es-module/test-esm-addon.mjs create mode 100644 test/es-module/test-esm-example-loader.js create mode 100644 test/es-module/test-esm-json.mjs create mode 100644 test/es-module/test-esm-named-exports.mjs create mode 100644 test/es-module/test-esm-resolve-hook.mjs create mode 100644 test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs create mode 100644 test/fixtures/es-module-loaders/example-loader.mjs create mode 100644 test/fixtures/es-module-loaders/js-as-esm.js create mode 100644 test/fixtures/es-module-loaders/js-loader.mjs diff --git a/.eslintrc.yaml b/.eslintrc.yaml index b68d2657465..d8768459c2e 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -11,7 +11,7 @@ parserOptions: ecmaVersion: 2017 overrides: - - files: ["doc/api/esm.md", "*.mjs"] + - files: ["doc/api/esm.md", "*.mjs", "test/es-module/test-esm-example-loader.js"] parserOptions: sourceType: module @@ -117,6 +117,7 @@ rules: keyword-spacing: error linebreak-style: [error, unix] max-len: [error, {code: 80, + ignorePattern: "^\/\/ Flags:", ignoreRegExpLiterals: true, ignoreUrls: true, tabWidth: 2}] diff --git a/doc/api/esm.md b/doc/api/esm.md index 7b684e99a27..bc25c88b9a8 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -98,4 +98,111 @@ fs.readFile('./foo.txt', (err, body) => { }); ``` +## Loader hooks + + + +To customize the default module resolution, loader hooks can optionally be +provided via a `--loader ./loader-name.mjs` argument to Node. + +When hooks are used they only apply to ES module loading and not to any +CommonJS modules loaded. + +### Resolve hook + +The resolve hook returns the resolved file URL and module format for a +given module specifier and parent file URL: + +```js +import url from 'url'; + +export async function resolve(specifier, parentModuleURL, defaultResolver) { + return { + url: new URL(specifier, parentModuleURL).href, + format: 'esm' + }; +} +``` + +The default NodeJS ES module resolution function is provided as a third +argument to the resolver for easy compatibility workflows. + +In addition to returning the resolved file URL value, the resolve hook also +returns a `format` property specifying the module format of the resolved +module. This can be one of `"esm"`, `"cjs"`, `"json"`, `"builtin"` or +`"addon"`. + +For example a dummy loader to load JavaScript restricted to browser resolution +rules with only JS file extension and Node builtin modules support could +be written: + +```js +import url from 'url'; +import path from 'path'; +import process from 'process'; + +const builtins = new Set( + Object.keys(process.binding('natives')).filter((str) => + /^(?!(?:internal|node|v8)\/)/.test(str)) +); +const JS_EXTENSIONS = new Set(['.js', '.mjs']); + +export function resolve(specifier, parentModuleURL/*, defaultResolve */) { + if (builtins.has(specifier)) { + return { + url: specifier, + format: 'builtin' + }; + } + if (/^\.{0,2}[/]/.test(specifier) !== true && !specifier.startsWith('file:')) { + // For node_modules support: + // return defaultResolve(specifier, parentModuleURL); + throw new Error( + `imports must begin with '/', './', or '../'; '${specifier}' does not`); + } + const resolved = new url.URL(specifier, parentModuleURL); + const ext = path.extname(resolved.pathname); + if (!JS_EXTENSIONS.has(ext)) { + throw new Error( + `Cannot load file with non-JavaScript file extension ${ext}.`); + } + return { + url: resolved.href, + format: 'esm' + }; +} +``` + +With this loader, running: + +``` +NODE_OPTIONS='--experimental-modules --loader ./custom-loader.mjs' node x.js +``` + +would load the module `x.js` as an ES module with relative resolution support +(with `node_modules` loading skipped in this example). + +### Dynamic instantiate hook + +To create a custom dynamic module that doesn't correspond to one of the +existing `format` interpretations, the `dynamicInstantiate` hook can be used. +This hook is called only for modules that return `format: "dynamic"` from +the `resolve` hook. + +```js +export async function dynamicInstantiate(url) { + return { + exports: ['customExportName'], + execute: (exports) => { + // get and set functions provided for pre-allocated export names + exports.customExportName.set('value'); + } + }; +} +``` + +With the list of module exports provided upfront, the `execute` function will +then be called at the exact point of module evalutation order for that module +in the import tree. + [Node.js EP for ES Modules]: https://github.com/nodejs/node-eps/blob/master/002-es-modules.md diff --git a/lib/internal/errors.js b/lib/internal/errors.js index f49bb2e542e..9a5551617d3 100755 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -328,6 +328,8 @@ E('ERR_TRANSFORM_WITH_LENGTH_0', E('ERR_UNESCAPED_CHARACTERS', (name) => `${name} contains unescaped characters`); E('ERR_UNKNOWN_ENCODING', 'Unknown encoding: %s'); +E('ERR_UNKNOWN_FILE_EXTENSION', 'Unknown file extension: %s'); +E('ERR_UNKNOWN_MODULE_FORMAT', 'Unknown module format: %s'); E('ERR_UNKNOWN_SIGNAL', 'Unknown signal: %s'); E('ERR_UNKNOWN_STDIN_TYPE', 'Unknown stdin file type'); E('ERR_UNKNOWN_STREAM_TYPE', 'Unknown stream file type'); diff --git a/lib/internal/loader/Loader.js b/lib/internal/loader/Loader.js index a409d397f85..57c70188d66 100644 --- a/lib/internal/loader/Loader.js +++ b/lib/internal/loader/Loader.js @@ -1,20 +1,21 @@ 'use strict'; -const { URL } = require('url'); const { getURLFromFilePath } = require('internal/url'); const { - getNamespaceOfModuleWrap + getNamespaceOfModuleWrap, + createDynamicModule } = require('internal/loader/ModuleWrap'); const ModuleMap = require('internal/loader/ModuleMap'); const ModuleJob = require('internal/loader/ModuleJob'); -const resolveRequestUrl = require('internal/loader/resolveRequestUrl'); +const ModuleRequest = require('internal/loader/ModuleRequest'); const errors = require('internal/errors'); +const debug = require('util').debuglog('esm'); function getBase() { try { - return getURLFromFilePath(`${process.cwd()}/`); + return getURLFromFilePath(`${process.cwd()}/`).href; } catch (e) { e.stack; // If the current working directory no longer exists. @@ -28,45 +29,75 @@ function getBase() { class Loader { constructor(base = getBase()) { this.moduleMap = new ModuleMap(); - if (typeof base !== 'undefined' && base instanceof URL !== true) { - throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'URL'); + if (typeof base !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'string'); } this.base = base; + this.resolver = ModuleRequest.resolve.bind(null); + this.dynamicInstantiate = undefined; } - async resolve(specifier) { - const request = resolveRequestUrl(this.base, specifier); - if (request.url.protocol !== 'file:') { - throw new errors.Error('ERR_INVALID_PROTOCOL', - request.url.protocol, 'file:'); - } - return request.url; + hook({ resolve = ModuleRequest.resolve, dynamicInstantiate }) { + this.resolver = resolve.bind(null); + this.dynamicInstantiate = dynamicInstantiate; } - async getModuleJob(dependentJob, specifier) { - if (!this.moduleMap.has(dependentJob.url)) { - throw new errors.Error('ERR_MISSING_MODULE', dependentJob.url); + async resolve(specifier, parentURL = this.base) { + if (typeof parentURL !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', + 'parentURL', 'string'); } - const request = await resolveRequestUrl(dependentJob.url, specifier); - const url = `${request.url}`; - if (this.moduleMap.has(url)) { - return this.moduleMap.get(url); + const { url, format } = await this.resolver(specifier, parentURL, + ModuleRequest.resolve); + + if (typeof format !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'format', + ['esm', 'cjs', 'builtin', 'addon', 'json']); } - const dependencyJob = new ModuleJob(this, request); - this.moduleMap.set(url, dependencyJob); - return dependencyJob; + if (typeof url !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string'); + } + + if (format === 'builtin') { + return { url: `node:${url}`, format }; + } + + if (format !== 'dynamic') { + if (!ModuleRequest.loaders.has(format)) { + throw new errors.Error('ERR_UNKNOWN_MODULE_FORMAT', format); + } + if (!url.startsWith('file:')) { + throw new errors.Error('ERR_INVALID_PROTOCOL', url, 'file:'); + } + } + + return { url, format }; } - async import(specifier) { - const request = await resolveRequestUrl(this.base, specifier); - const url = `${request.url}`; - let job; - if (this.moduleMap.has(url)) { - job = this.moduleMap.get(url); - } else { - job = new ModuleJob(this, request); + async getModuleJob(specifier, parentURL = this.base) { + const { url, format } = await this.resolve(specifier, parentURL); + let job = this.moduleMap.get(url); + if (job === undefined) { + let loaderInstance; + if (format === 'dynamic') { + loaderInstance = async (url) => { + const { exports, execute } = await this.dynamicInstantiate(url); + return createDynamicModule(exports, url, (reflect) => { + debug(`Loading custom loader ${url}`); + execute(reflect.exports); + }); + }; + } else { + loaderInstance = ModuleRequest.loaders.get(format); + } + job = new ModuleJob(this, url, loaderInstance); this.moduleMap.set(url, job); } + return job; + } + + async import(specifier, parentURL = this.base) { + const job = await this.getModuleJob(specifier, parentURL); const module = await job.run(); return getNamespaceOfModuleWrap(module); } diff --git a/lib/internal/loader/ModuleJob.js b/lib/internal/loader/ModuleJob.js index db4cb6ae5c5..04d6111b87f 100644 --- a/lib/internal/loader/ModuleJob.js +++ b/lib/internal/loader/ModuleJob.js @@ -2,51 +2,40 @@ const { SafeSet, SafePromise } = require('internal/safe_globals'); const resolvedPromise = SafePromise.resolve(); -const resolvedArrayPromise = SafePromise.resolve([]); -const { ModuleWrap } = require('internal/loader/ModuleWrap'); -const NOOP = () => { /* No-op */ }; class ModuleJob { /** * @param {module: ModuleWrap?, compiled: Promise} moduleProvider */ - constructor(loader, moduleProvider, url) { - this.url = `${moduleProvider.url}`; - this.moduleProvider = moduleProvider; + constructor(loader, url, moduleProvider) { this.loader = loader; this.error = null; this.hadError = false; - if (moduleProvider instanceof ModuleWrap !== true) { - // linked == promise for dependency jobs, with module populated, - // module wrapper linked - this.modulePromise = this.moduleProvider.createModule(); - this.module = undefined; - const linked = async () => { - const dependencyJobs = []; - this.module = await this.modulePromise; - this.module.link(async (dependencySpecifier) => { - const dependencyJobPromise = - this.loader.getModuleJob(this, dependencySpecifier); - dependencyJobs.push(dependencyJobPromise); - const dependencyJob = await dependencyJobPromise; - return dependencyJob.modulePromise; - }); - return SafePromise.all(dependencyJobs); - }; - this.linked = linked(); + // linked == promise for dependency jobs, with module populated, + // module wrapper linked + this.moduleProvider = moduleProvider; + this.modulePromise = this.moduleProvider(url); + this.module = undefined; + this.reflect = undefined; + const linked = async () => { + const dependencyJobs = []; + ({ module: this.module, + reflect: this.reflect } = await this.modulePromise); + this.module.link(async (dependencySpecifier) => { + const dependencyJobPromise = + this.loader.getModuleJob(dependencySpecifier, url); + dependencyJobs.push(dependencyJobPromise); + const dependencyJob = await dependencyJobPromise; + return (await dependencyJob.modulePromise).module; + }); + return SafePromise.all(dependencyJobs); + }; + this.linked = linked(); - // instantiated == deep dependency jobs wrappers instantiated, - //module wrapper instantiated - this.instantiated = undefined; - } else { - const getModuleProvider = async () => moduleProvider; - this.modulePromise = getModuleProvider(); - this.moduleProvider = { finish: NOOP }; - this.module = moduleProvider; - this.linked = resolvedArrayPromise; - this.instantiated = this.modulePromise; - } + // instantiated == deep dependency jobs wrappers instantiated, + // module wrapper instantiated + this.instantiated = undefined; } instantiate() { diff --git a/lib/internal/loader/ModuleRequest.js b/lib/internal/loader/ModuleRequest.js new file mode 100644 index 00000000000..88e48ae9d3e --- /dev/null +++ b/lib/internal/loader/ModuleRequest.js @@ -0,0 +1,122 @@ +'use strict'; + +const fs = require('fs'); +const internalCJSModule = require('internal/module'); +const internalURLModule = require('internal/url'); +const internalFS = require('internal/fs'); +const NativeModule = require('native_module'); +const { extname, _makeLong } = require('path'); +const { URL } = require('url'); +const { realpathSync } = require('fs'); +const preserveSymlinks = !!process.binding('config').preserveSymlinks; +const { + ModuleWrap, + createDynamicModule +} = require('internal/loader/ModuleWrap'); +const errors = require('internal/errors'); + +const search = require('internal/loader/search'); +const asyncReadFile = require('util').promisify(require('fs').readFile); +const debug = require('util').debuglog('esm'); + +const realpathCache = new Map(); + +const loaders = new Map(); +exports.loaders = loaders; + +// Strategy for loading a standard JavaScript module +loaders.set('esm', async (url) => { + const source = `${await asyncReadFile(new URL(url))}`; + debug(`Loading StandardModule ${url}`); + return { + module: new ModuleWrap(internalCJSModule.stripShebang(source), url), + reflect: undefined + }; +}); + +// Strategy for loading a node-style CommonJS module +loaders.set('cjs', async (url) => { + return createDynamicModule(['default'], url, (reflect) => { + debug(`Loading CJSModule ${url}`); + const CJSModule = require('module'); + const pathname = internalURLModule.getPathFromURL(new URL(url)); + CJSModule._load(pathname); + }); +}); + +// Strategy for loading a node builtin CommonJS module that isn't +// through normal resolution +loaders.set('builtin', async (url) => { + return createDynamicModule(['default'], url, (reflect) => { + debug(`Loading BuiltinModule ${url}`); + const exports = NativeModule.require(url.substr(5)); + reflect.exports.default.set(exports); + }); +}); + +loaders.set('addon', async (url) => { + const ctx = createDynamicModule(['default'], url, (reflect) => { + debug(`Loading NativeModule ${url}`); + const module = { exports: {} }; + const pathname = internalURLModule.getPathFromURL(new URL(url)); + process.dlopen(module, _makeLong(pathname)); + reflect.exports.default.set(module.exports); + }); + return ctx; +}); + +loaders.set('json', async (url) => { + return createDynamicModule(['default'], url, (reflect) => { + debug(`Loading JSONModule ${url}`); + const pathname = internalURLModule.getPathFromURL(new URL(url)); + const content = fs.readFileSync(pathname, 'utf8'); + try { + const exports = JSON.parse(internalCJSModule.stripBOM(content)); + reflect.exports.default.set(exports); + } catch (err) { + err.message = pathname + ': ' + err.message; + throw err; + } + }); +}); + +exports.resolve = (specifier, parentURL) => { + if (NativeModule.nonInternalExists(specifier)) { + return { + url: specifier, + format: 'builtin' + }; + } + + let url = search(specifier, parentURL); + + if (url.protocol !== 'file:') { + throw new errors.Error('ERR_INVALID_PROTOCOL', + url.protocol, 'file:'); + } + + if (!preserveSymlinks) { + const real = realpathSync(internalURLModule.getPathFromURL(url), { + [internalFS.realpathCacheKey]: realpathCache + }); + const old = url; + url = internalURLModule.getURLFromFilePath(real); + url.search = old.search; + url.hash = old.hash; + } + + const ext = extname(url.pathname); + switch (ext) { + case '.mjs': + return { url: `${url}`, format: 'esm' }; + case '.json': + return { url: `${url}`, format: 'json' }; + case '.node': + return { url: `${url}`, format: 'addon' }; + case '.js': + return { url: `${url}`, format: 'cjs' }; + default: + throw new errors.Error('ERR_UNKNOWN_FILE_EXTENSION', + internalURLModule.getPathFromURL(url)); + } +}; diff --git a/lib/internal/loader/resolveRequestUrl.js b/lib/internal/loader/resolveRequestUrl.js deleted file mode 100644 index 2245064bfe4..00000000000 --- a/lib/internal/loader/resolveRequestUrl.js +++ /dev/null @@ -1,104 +0,0 @@ -'use strict'; - -const { URL } = require('url'); -const internalCJSModule = require('internal/module'); -const internalURLModule = require('internal/url'); -const internalFS = require('internal/fs'); -const NativeModule = require('native_module'); -const { extname } = require('path'); -const { realpathSync } = require('fs'); -const preserveSymlinks = !!process.binding('config').preserveSymlinks; -const { - ModuleWrap, - createDynamicModule -} = require('internal/loader/ModuleWrap'); -const errors = require('internal/errors'); - -const search = require('internal/loader/search'); -const asyncReadFile = require('util').promisify(require('fs').readFile); -const debug = require('util').debuglog('esm'); - -const realpathCache = new Map(); - -class ModuleRequest { - constructor(url) { - this.url = url; - } -} -Object.setPrototypeOf(ModuleRequest.prototype, null); - -// Strategy for loading a standard JavaScript module -class StandardModuleRequest extends ModuleRequest { - async createModule() { - const source = `${await asyncReadFile(this.url)}`; - debug(`Loading StandardModule ${this.url}`); - return new ModuleWrap(internalCJSModule.stripShebang(source), - `${this.url}`); - } -} - -// Strategy for loading a node-style CommonJS module -class CJSModuleRequest extends ModuleRequest { - async createModule() { - const ctx = createDynamicModule(['default'], this.url, (reflect) => { - debug(`Loading CJSModule ${this.url.pathname}`); - const CJSModule = require('module'); - const pathname = internalURLModule.getPathFromURL(this.url); - CJSModule._load(pathname); - }); - this.finish = (module) => { - ctx.reflect.exports.default.set(module.exports); - }; - return ctx.module; - } -} - -// Strategy for loading a node builtin CommonJS module that isn't -// through normal resolution -class NativeModuleRequest extends CJSModuleRequest { - async createModule() { - const ctx = createDynamicModule(['default'], this.url, (reflect) => { - debug(`Loading NativeModule ${this.url.pathname}`); - const exports = require(this.url.pathname); - reflect.exports.default.set(exports); - }); - return ctx.module; - } -} - -const normalizeBaseURL = (baseURLOrString) => { - if (baseURLOrString instanceof URL) return baseURLOrString; - if (typeof baseURLOrString === 'string') return new URL(baseURLOrString); - return undefined; -}; - -const resolveRequestUrl = (baseURLOrString, specifier) => { - if (NativeModule.nonInternalExists(specifier)) { - return new NativeModuleRequest(new URL(`node:${specifier}`)); - } - - const baseURL = normalizeBaseURL(baseURLOrString); - let url = search(specifier, baseURL); - - if (url.protocol !== 'file:') { - throw new errors.Error('ERR_INVALID_PROTOCOL', url.protocol, 'file:'); - } - - if (!preserveSymlinks) { - const real = realpathSync(internalURLModule.getPathFromURL(url), { - [internalFS.realpathCacheKey]: realpathCache - }); - const old = url; - url = internalURLModule.getURLFromFilePath(real); - url.search = old.search; - url.hash = old.hash; - } - - const ext = extname(url.pathname); - if (ext === '.mjs') { - return new StandardModuleRequest(url); - } - - return new CJSModuleRequest(url); -}; -module.exports = resolveRequestUrl; diff --git a/lib/internal/loader/search.js b/lib/internal/loader/search.js index f0ec34ae4e7..8b1a3e0f83f 100644 --- a/lib/internal/loader/search.js +++ b/lib/internal/loader/search.js @@ -6,12 +6,10 @@ const errors = require('internal/errors'); const { resolve } = process.binding('module_wrap'); module.exports = (target, base) => { - target = `${target}`; if (base === undefined) { // We cannot search without a base. throw new errors.Error('ERR_MISSING_MODULE', target); } - base = `${base}`; try { return resolve(target, base); } catch (e) { diff --git a/lib/module.js b/lib/module.js index 6693bedeb1c..73f3cc8dd8c 100644 --- a/lib/module.js +++ b/lib/module.js @@ -40,7 +40,7 @@ const errors = require('internal/errors'); const Loader = require('internal/loader/Loader'); const ModuleJob = require('internal/loader/ModuleJob'); const { createDynamicModule } = require('internal/loader/ModuleWrap'); -const ESMLoader = new Loader(); +let ESMLoader; function stat(filename) { filename = path.toNamespacedPath(filename); @@ -424,29 +424,31 @@ Module._load = function(request, parent, isMain) { var filename = null; if (isMain) { - let err; try { filename = Module._resolveFilename(request, parent, isMain); } catch (e) { // try to keep stack e.stack; - err = e; + throw e; } if (experimentalModules) { - if (filename === null || /\.mjs$/.test(filename)) { - try { - ESMLoader.import(getURLFromFilePath(filename).href).catch((e) => { - console.error(e); - process.exit(1); - }); - return; - } catch (e) { - // well, it isn't ESM + (async () => { + // loader setup + if (!ESMLoader) { + ESMLoader = new Loader(); + const userLoader = process.binding('config').userLoader; + if (userLoader) { + const hooks = await new Loader().import(userLoader); + ESMLoader.hook(hooks); + } } - } - } - if (err) { - throw err; + await ESMLoader.import(getURLFromFilePath(filename).href); + })() + .catch((e) => { + console.error(e); + process.exit(1); + }); + return; } } else { filename = Module._resolveFilename(request, parent, isMain); @@ -521,16 +523,18 @@ Module.prototype.load = function(filename) { Module._extensions[extension](this, filename); this.loaded = true; - if (experimentalModules) { + if (ESMLoader) { const url = getURLFromFilePath(filename); - if (ESMLoader.moduleMap.has(`${url}`) !== true) { + const urlString = `${url}`; + if (ESMLoader.moduleMap.has(urlString) !== true) { const ctx = createDynamicModule(['default'], url); ctx.reflect.exports.default.set(this.exports); - ESMLoader.moduleMap.set(`${url}`, - new ModuleJob(ESMLoader, ctx.module)); + ESMLoader.moduleMap.set(urlString, + new ModuleJob(ESMLoader, url, async () => ctx)); } else { - ESMLoader.moduleMap.get(`${url}`).moduleProvider.finish( - Module._cache[filename]); + const job = ESMLoader.moduleMap.get(urlString); + if (job.reflect) + job.reflect.exports.default.set(this.exports); } } }; diff --git a/node.gyp b/node.gyp index 4fed35c49af..66949aa3b8b 100644 --- a/node.gyp +++ b/node.gyp @@ -103,7 +103,7 @@ 'lib/internal/loader/ModuleMap.js', 'lib/internal/loader/ModuleJob.js', 'lib/internal/loader/ModuleWrap.js', - 'lib/internal/loader/resolveRequestUrl.js', + 'lib/internal/loader/ModuleRequest.js', 'lib/internal/loader/search.js', 'lib/internal/safe_globals.js', 'lib/internal/net.js', diff --git a/src/node.cc b/src/node.cc index 31aa7a50fe3..a73d8b58072 100644 --- a/src/node.cc +++ b/src/node.cc @@ -231,6 +231,11 @@ bool config_preserve_symlinks = false; // that is used by lib/module.js bool config_experimental_modules = false; +// Set in node.cc by ParseArgs when --loader is used. +// Used in node_config.cc to set a constant on process.binding('config') +// that is used by lib/internal/bootstrap_node.js +std::string config_userland_loader; // NOLINT(runtime/string) + // Set by ParseArgs when --pending-deprecation or NODE_PENDING_DEPRECATION // is used. bool config_pending_deprecation = false; @@ -3714,7 +3719,6 @@ static void RawDebug(const FunctionCallbackInfo& args) { fflush(stderr); } - void LoadEnvironment(Environment* env) { HandleScope handle_scope(env->isolate()); @@ -3737,6 +3741,7 @@ void LoadEnvironment(Environment* env) { } // The bootstrap_node.js file returns a function 'f' CHECK(f_value->IsFunction()); + Local f = Local::Cast(f_value); // Add a reference to the global object @@ -3776,6 +3781,7 @@ void LoadEnvironment(Environment* env) { // who do not like how bootstrap_node.js sets up the module system but do // like Node's I/O bindings may want to replace 'f' with their own function. Local arg = env->process_object(); + auto ret = f->Call(env->context(), Null(env->isolate()), 1, &arg); // If there was an error during bootstrap then it was either handled by the // FatalException handler or it's unrecoverable (e.g. max call stack @@ -3950,6 +3956,8 @@ static void CheckIfAllowedInEnv(const char* exe, bool is_env, "--no-warnings", "--napi-modules", "--expose-http2", // keep as a non-op through v9.x + "--experimental-modules", + "--loader", "--trace-warnings", "--redirect-warnings", "--trace-sync-io", @@ -4112,6 +4120,19 @@ static void ParseArgs(int* argc, config_preserve_symlinks = true; } else if (strcmp(arg, "--experimental-modules") == 0) { config_experimental_modules = true; + } else if (strcmp(arg, "--loader") == 0) { + const char* module = argv[index + 1]; + if (!config_experimental_modules) { + fprintf(stderr, "%s: %s requires --experimental-modules be enabled\n", + argv[0], arg); + exit(9); + } + if (module == nullptr) { + fprintf(stderr, "%s: %s requires an argument\n", argv[0], arg); + exit(9); + } + args_consumed += 1; + config_userland_loader = module; } else if (strcmp(arg, "--prof-process") == 0) { prof_process = true; short_circuit = true; diff --git a/src/node_config.cc b/src/node_config.cc index 3f0dbbee8b5..38ce2a47bb0 100644 --- a/src/node_config.cc +++ b/src/node_config.cc @@ -65,8 +65,18 @@ static void InitConfig(Local target, if (config_preserve_symlinks) READONLY_BOOLEAN_PROPERTY("preserveSymlinks"); - if (config_experimental_modules) + if (config_experimental_modules) { READONLY_BOOLEAN_PROPERTY("experimentalModules"); + if (!config_userland_loader.empty()) { + target->DefineOwnProperty( + context, + FIXED_ONE_BYTE_STRING(isolate, "userLoader"), + String::NewFromUtf8(isolate, + config_userland_loader.data(), + v8::NewStringType::kNormal).ToLocalChecked(), + ReadOnly).FromJust(); + } + } if (config_pending_deprecation) READONLY_BOOLEAN_PROPERTY("pendingDeprecation"); diff --git a/src/node_internals.h b/src/node_internals.h index 13e6b12d1d9..7c4f7a6a7fb 100644 --- a/src/node_internals.h +++ b/src/node_internals.h @@ -91,6 +91,11 @@ extern bool config_preserve_symlinks; // that is used by lib/module.js extern bool config_experimental_modules; +// Set in node.cc by ParseArgs when --loader is used. +// Used in node_config.cc to set a constant on process.binding('config') +// that is used by lib/internal/bootstrap_node.js +extern std::string config_userland_loader; + // Set in node.cc by ParseArgs when --expose-internals or --expose_internals is // used. // Used in node_config.cc to set a constant on process.binding('config') diff --git a/test/es-module/json.json b/test/es-module/json.json new file mode 100644 index 00000000000..8288d42e2bc --- /dev/null +++ b/test/es-module/json.json @@ -0,0 +1,3 @@ +{ + "val": 42 +} diff --git a/test/es-module/test-esm-addon.mjs b/test/es-module/test-esm-addon.mjs new file mode 100644 index 00000000000..90ed9ffa50b --- /dev/null +++ b/test/es-module/test-esm-addon.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +import assert from 'assert'; +import binding from '../addons/hello-world/build/Release/binding.node'; +assert.strictEqual(binding.hello(), 'world'); +console.log('binding.hello() =', binding.hello()); diff --git a/test/es-module/test-esm-example-loader.js b/test/es-module/test-esm-example-loader.js new file mode 100644 index 00000000000..9d1348292cd --- /dev/null +++ b/test/es-module/test-esm-example-loader.js @@ -0,0 +1,6 @@ +// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/example-loader.mjs +/* eslint-disable required-modules */ +import assert from 'assert'; +import ok from './test-esm-ok.mjs'; + +assert(ok); diff --git a/test/es-module/test-esm-json.mjs b/test/es-module/test-esm-json.mjs new file mode 100644 index 00000000000..864f4964def --- /dev/null +++ b/test/es-module/test-esm-json.mjs @@ -0,0 +1,8 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ +import assert from 'assert'; +import ok from './test-esm-ok.mjs'; +import json from './json.json'; + +assert(ok); +assert.strictEqual(json.val, 42); diff --git a/test/es-module/test-esm-named-exports.mjs b/test/es-module/test-esm-named-exports.mjs new file mode 100644 index 00000000000..89a29007026 --- /dev/null +++ b/test/es-module/test-esm-named-exports.mjs @@ -0,0 +1,8 @@ +// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs +/* eslint-disable required-modules */ +import { readFile } from 'fs'; +import assert from 'assert'; +import ok from './test-esm-ok.mjs'; + +assert(ok); +assert(readFile); diff --git a/test/es-module/test-esm-resolve-hook.mjs b/test/es-module/test-esm-resolve-hook.mjs new file mode 100644 index 00000000000..dd7ac80bec4 --- /dev/null +++ b/test/es-module/test-esm-resolve-hook.mjs @@ -0,0 +1,8 @@ +// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/js-loader.mjs +/* eslint-disable required-modules */ +import { namedExport } from '../fixtures/es-module-loaders/js-as-esm.js'; +import assert from 'assert'; +import ok from './test-esm-ok.mjs'; + +assert(ok); +assert(namedExport); diff --git a/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs b/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs new file mode 100644 index 00000000000..0734003802e --- /dev/null +++ b/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs @@ -0,0 +1,29 @@ +import module from 'module'; + +const builtins = new Set( + Object.keys(process.binding('natives')).filter(str => + /^(?!(?:internal|node|v8)\/)/.test(str)) +) + +export function resolve (specifier, base, defaultResolver) { + if (builtins.has(specifier)) { + return { + url: `node:${specifier}`, + format: 'dynamic' + }; + } + return defaultResolver(specifier, base); +} + +export async function dynamicInstantiate (url) { + const builtinInstance = module._load(url.substr(5)); + const builtinExports = ['default', ...Object.keys(builtinInstance)]; + return { + exports: builtinExports, + execute: exports => { + for (let name of builtinExports) + exports[name].set(builtinInstance[name]); + exports.default.set(builtinInstance); + } + }; +} diff --git a/test/fixtures/es-module-loaders/example-loader.mjs b/test/fixtures/es-module-loaders/example-loader.mjs new file mode 100644 index 00000000000..771273a8d86 --- /dev/null +++ b/test/fixtures/es-module-loaders/example-loader.mjs @@ -0,0 +1,34 @@ +import url from 'url'; +import path from 'path'; +import process from 'process'; + +const builtins = new Set( + Object.keys(process.binding('natives')).filter((str) => + /^(?!(?:internal|node|v8)\/)/.test(str)) +); +const JS_EXTENSIONS = new Set(['.js', '.mjs']); + +export function resolve(specifier, parentModuleURL/*, defaultResolve */) { + if (builtins.has(specifier)) { + return { + url: specifier, + format: 'builtin' + }; + } + if (/^\.{0,2}[/]/.test(specifier) !== true && !specifier.startsWith('file:')) { + // For node_modules support: + // return defaultResolve(specifier, parentModuleURL); + throw new Error( + `imports must begin with '/', './', or '../'; '${specifier}' does not`); + } + const resolved = new url.URL(specifier, parentModuleURL); + const ext = path.extname(resolved.pathname); + if (!JS_EXTENSIONS.has(ext)) { + throw new Error( + `Cannot load file with non-JavaScript file extension ${ext}.`); + } + return { + url: resolved.href, + format: 'esm' + }; +} diff --git a/test/fixtures/es-module-loaders/js-as-esm.js b/test/fixtures/es-module-loaders/js-as-esm.js new file mode 100644 index 00000000000..b4d2741b2fc --- /dev/null +++ b/test/fixtures/es-module-loaders/js-as-esm.js @@ -0,0 +1 @@ +export const namedExport = 'named-export'; diff --git a/test/fixtures/es-module-loaders/js-loader.mjs b/test/fixtures/es-module-loaders/js-loader.mjs new file mode 100644 index 00000000000..79d9774c1d7 --- /dev/null +++ b/test/fixtures/es-module-loaders/js-loader.mjs @@ -0,0 +1,19 @@ +import _url from 'url'; +const builtins = new Set( + Object.keys(process.binding('natives')).filter(str => + /^(?!(?:internal|node|v8)\/)/.test(str)) +) +export function resolve (specifier, base) { + if (builtins.has(specifier)) { + return { + url: specifier, + format: 'builtin' + }; + } + // load all dependencies as esm, regardless of file extension + const url = new _url.URL(specifier, base).href; + return { + url, + format: 'esm' + }; +}