module: refactor modules bootstrap

PR-URL: https://github.com/nodejs/node/pull/29937
Reviewed-By: Myles Borins <myles.borins@gmail.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>

doc/api/esm.md

@@ -886,13 +886,13 @@ _isMain_ is **true** when resolving the Node.js application entry point.
 >    1. Throw a _Module Not Found_ error.
 > 1. If _pjson.exports_ is not **null** or **undefined**, then
 >    1. If _pjson.exports_ is a String or Array, then
->       1. Return _PACKAGE_EXPORTS_TARGET_RESOLVE(packageURL, pjson.exports,
+>       1. Return **PACKAGE_EXPORTS_TARGET_RESOLVE**(_packageURL_,
->          "")_.
+>          _pjson.exports_, "")_.
 >    1. If _pjson.exports is an Object, then
 >       1. If _pjson.exports_ contains a _"."_ property, then
 >          1. Let _mainExport_ be the _"."_ property in _pjson.exports_.
->          1. Return _PACKAGE_EXPORTS_TARGET_RESOLVE(packageURL, mainExport,
+>          1. Return **PACKAGE_EXPORTS_TARGET_RESOLVE**(_packageURL_,
->             "")_.
+>             _mainExport_, "")_.
 > 1. If _pjson.main_ is a String, then
 >    1. Let _resolvedMain_ be the URL resolution of _packageURL_, "/", and
 >       _pjson.main_.

lib/internal/bootstrap/loaders.js

@@ -220,7 +220,10 @@ NativeModule.prototype.compileForPublicLoader = function(needToSyncExports) {
   this.compile();
   if (needToSyncExports) {
     if (!this.exportKeys) {
-      this.exportKeys = Object.keys(this.exports);
+      // When using --expose-internals, we do not want to reflect the named
+      // exports from core modules as this can trigger unnecessary getters.
+      const internal = this.id.startsWith('internal/');
+      this.exportKeys = internal ? [] : Object.keys(this.exports);
     }
     this.getESMFacade();
     this.syncExports();

lib/internal/bootstrap/pre_execution.js

@@ -5,6 +5,7 @@ const { Object, SafeWeakMap } = primordials;
 const { getOptionValue } = require('internal/options');
 const { Buffer } = require('buffer');
 const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes;
+const path = require('path');
 
 function prepareMainThreadExecution(expandArgv1 = false) {
   // Patch the process object with legacy properties and normalizations
@@ -404,7 +405,6 @@ function initializeESMLoader() {
         'The ESM module loader is experimental.',
         'ExperimentalWarning', undefined);
     }
-
     const {
       setImportModuleDynamicallyCallback,
       setInitializeImportMetaObjectCallback
@@ -414,14 +414,6 @@ function initializeESMLoader() {
     // track of for different ESM modules.
     setInitializeImportMetaObjectCallback(esm.initializeImportMetaObject);
     setImportModuleDynamicallyCallback(esm.importModuleDynamicallyCallback);
-    const userLoader = getOptionValue('--experimental-loader');
-    // If --experimental-loader is specified, create a loader with user hooks.
-    // Otherwise create the default loader.
-    if (userLoader) {
-      const { emitExperimentalWarning } = require('internal/util');
-      emitExperimentalWarning('--experimental-loader');
-    }
-    esm.initializeLoader(process.cwd(), userLoader);
   }
 }
 
@@ -446,11 +438,70 @@ function loadPreloadModules() {
   }
 }
 
+function resolveMainPath(main) {
+  const { toRealPath, Module: CJSModule } =
+      require('internal/modules/cjs/loader');
+
+  // Note extension resolution for the main entry point can be deprecated in a
+  // future major.
+  let mainPath = CJSModule._findPath(path.resolve(main), null, true);
+  if (!mainPath)
+    return;
+
+  const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main');
+  if (!preserveSymlinksMain)
+    mainPath = toRealPath(mainPath);
+
+  return mainPath;
+}
+
+function shouldUseESMLoader(mainPath) {
+  const experimentalModules = getOptionValue('--experimental-modules');
+  if (!experimentalModules)
+    return false;
+  const userLoader = getOptionValue('--experimental-loader');
+  if (userLoader)
+    return true;
+  // Determine the module format of the main
+  if (mainPath && mainPath.endsWith('.mjs'))
+    return true;
+  if (!mainPath || mainPath.endsWith('.cjs'))
+    return false;
+  const { readPackageScope } = require('internal/modules/cjs/loader');
+  const pkg = readPackageScope(mainPath);
+  return pkg && pkg.data.type === 'module';
+}
+
+function runMainESM(mainPath) {
+  const esmLoader = require('internal/process/esm_loader');
+  const { pathToFileURL } = require('internal/url');
+  const { hasUncaughtExceptionCaptureCallback } =
+      require('internal/process/execution');
+  return esmLoader.initializeLoader().then(() => {
+    const main = path.isAbsolute(mainPath) ?
+      pathToFileURL(mainPath).href : mainPath;
+    return esmLoader.ESMLoader.import(main).catch((e) => {
+      if (hasUncaughtExceptionCaptureCallback()) {
+        process._fatalException(e);
+        return;
+      }
+      internalBinding('errors').triggerUncaughtException(
+        e,
+        true /* fromPromise */
+      );
+    });
+  });
+}
+
+
 module.exports = {
   patchProcessObject,
+  resolveMainPath,
+  runMainESM,
   setupCoverageHooks,
   setupWarningHandler,
   setupDebugEnv,
+  shouldUseESMLoader,
   prepareMainThreadExecution,
   initializeDeprecations,
   initializeESMLoader,

lib/internal/main/run_main_module.js

@@ -10,8 +10,7 @@ const CJSModule = require('internal/modules/cjs/loader').Module;
 
 markBootstrapComplete();
 
-// Note: this actually tries to run the module as a ESM first if
+// Note: this loads the module through the ESM loader if
-// --experimental-modules is on.
+// --experimental-loader is provided or --experimental-modules is on
-// TODO(joyeecheung): can we move that logic to here? Note that this
+// and the module is determined to be an ES module
-// is an undocumented method available via `require('module').runMain`
+CJSModule.runMain(process.argv[1]);
-CJSModule.runMain();

lib/internal/main/worker_thread.js

@@ -140,8 +140,9 @@ port.on('message', (message) => {
       const { evalScript } = require('internal/process/execution');
       evalScript('[worker eval]', filename);
     } else {
-      process.argv[1] = filename; // script filename
+      // script filename
-      require('module').runMain();
+      const CJSModule = require('internal/modules/cjs/loader').Module;
+      CJSModule.runMain(process.argv[1] = filename);
     }
   } else if (message.type === STDIO_PAYLOAD) {
     const { stream, chunk, encoding } = message;

lib/internal/modules/cjs/loader.js

@@ -69,9 +69,14 @@ const {
   ERR_REQUIRE_ESM
 } = require('internal/errors').codes;
 const { validateString } = require('internal/validators');
+const {
+  resolveMainPath,
+  shouldUseESMLoader,
+  runMainESM
+} = require('internal/bootstrap/pre_execution');
 const pendingDeprecation = getOptionValue('--pending-deprecation');
 
-module.exports = { wrapSafe, Module };
+module.exports = { wrapSafe, Module, toRealPath, readPackageScope };
 
 let asyncESM, ModuleJob, ModuleWrap, kInstantiated;
 
@@ -810,6 +815,10 @@ Module.prototype.load = function(filename) {
   this.paths = Module._nodeModulePaths(path.dirname(filename));
 
   const extension = findLongestRegisteredExtension(filename);
+  // allow .mjs to be overridden
+  if (filename.endsWith('.mjs') && !Module._extensions['.mjs']) {
+    throw new ERR_REQUIRE_ESM(filename);
+  }
   Module._extensions[extension](this, filename);
   this.loaded = true;
 
@@ -823,14 +832,19 @@ Module.prototype.load = function(filename) {
     if (module !== undefined && module.module !== undefined) {
       if (module.module.getStatus() >= kInstantiated)
         module.module.setExport('default', exports);
-    } else { // preemptively cache
+    } else {
+      // Preemptively cache
+      // We use a function to defer promise creation for async hooks.
       ESMLoader.moduleMap.set(
         url,
-        new ModuleJob(ESMLoader, url, () =>
+        // Module job creation will start promises.
+        // We make it a function to lazily trigger those promises
+        // for async hooks compatibility.
+        () => new ModuleJob(ESMLoader, url, () =>
           new ModuleWrap(url, undefined, ['default'], function() {
             this.setExport('default', exports);
           })
-        )
+        , false /* isMain */, false /* inspectBrk */)
       );
     }
   }
@@ -859,7 +873,7 @@ Module.prototype.require = function(id) {
 var resolvedArgv;
 let hasPausedEntry = false;
 
-function wrapSafe(filename, content) {
+function wrapSafe(filename, content, cjsModuleInstance) {
   if (patched) {
     const wrapper = Module.wrap(content);
     return vm.runInThisContext(wrapper, {
@@ -867,7 +881,7 @@ function wrapSafe(filename, content) {
       lineOffset: 0,
       displayErrors: true,
       importModuleDynamically: experimentalModules ? async (specifier) => {
-        const loader = await asyncESM.loaderPromise;
+        const loader = asyncESM.ESMLoader;
         return loader.import(specifier, normalizeReferrerURL(filename));
       } : undefined,
     });
@@ -892,9 +906,8 @@ function wrapSafe(filename, content) {
       ]
     );
   } catch (err) {
-    if (experimentalModules) {
+    if (experimentalModules && process.mainModule === cjsModuleInstance)
       enrichCJSError(err);
-    }
     throw err;
   }
 
@@ -902,7 +915,7 @@ function wrapSafe(filename, content) {
     const { callbackMap } = internalBinding('module_wrap');
     callbackMap.set(compiled.cacheKey, {
       importModuleDynamically: async (specifier) => {
-        const loader = await asyncESM.loaderPromise;
+        const loader = asyncESM.ESMLoader;
         return loader.import(specifier, normalizeReferrerURL(filename));
       }
     });
@@ -925,7 +938,7 @@ Module.prototype._compile = function(content, filename) {
   }
 
   maybeCacheSourceMap(filename, content, this);
-  const compiledWrapper = wrapSafe(filename, content);
+  const compiledWrapper = wrapSafe(filename, content, this);
 
   var inspectorWrapper = null;
   if (getOptionValue('--inspect-brk') && process._eval == null) {
@@ -981,7 +994,11 @@ Module._extensions['.js'] = function(module, filename) {
           'files in that package scope as ES modules.\nInstead rename ' +
           `${basename} to end in .cjs, change the requiring code to use ` +
           'import(), or remove "type": "module" from ' +
-          `${path.resolve(pkg.path, 'package.json')}.`
+          `${path.resolve(pkg.path, 'package.json')}.`,
+          undefined,
+          undefined,
+          undefined,
+          true
         );
         warnRequireESM = false;
       }
@@ -1024,26 +1041,15 @@ Module._extensions['.node'] = function(module, filename) {
   return process.dlopen(module, path.toNamespacedPath(filename));
 };
 
-Module._extensions['.mjs'] = function(module, filename) {
-  throw new ERR_REQUIRE_ESM(filename);
-};
-
 // Bootstrap main module.
-Module.runMain = function() {
+Module.runMain = function(main = process.argv[1]) {
-  // Load the main module--the command line argument.
+  const resolvedMain = resolveMainPath(main);
-  if (experimentalModules) {
+  const useESMLoader = shouldUseESMLoader(resolvedMain);
-    asyncESM.loaderPromise.then((loader) => {
+  if (useESMLoader) {
-      return loader.import(pathToFileURL(process.argv[1]).href);
+    runMainESM(resolvedMain || main);
-    })
+  } else {
-    .catch((e) => {
+    Module._load(main, null, true);
-      internalBinding('errors').triggerUncaughtException(
-        e,
-        true /* fromPromise */
-      );
-    });
-    return;
   }
-  Module._load(process.argv[1], null, true);
 };
 
 function createRequireFromPath(filename) {
@@ -1164,7 +1170,7 @@ Module.Module = Module;
 
 // We have to load the esm things after module.exports!
 if (experimentalModules) {
-  asyncESM = require('internal/process/esm_loader');
   ModuleJob = require('internal/modules/esm/module_job');
+  asyncESM = require('internal/process/esm_loader');
   ({ ModuleWrap, kInstantiated } = internalBinding('module_wrap'));
 }

lib/internal/modules/esm/loader.js

@@ -22,6 +22,7 @@ const createDynamicModule = require(
   'internal/modules/esm/create_dynamic_module');
 const { translators } = require('internal/modules/esm/translators');
 const { ModuleWrap } = internalBinding('module_wrap');
+const { getOptionValue } = require('internal/options');
 
 const debug = require('internal/util/debuglog').debuglog('esm');
 
@@ -118,7 +119,7 @@ class Loader {
     url = pathToFileURL(`${process.cwd()}/[eval${++this.evalIndex}]`).href
   ) {
     const evalInstance = (url) => new ModuleWrap(url, undefined, source, 0, 0);
-    const job = new ModuleJob(this, url, evalInstance, false);
+    const job = new ModuleJob(this, url, evalInstance, false, false);
     this.moduleMap.set(url, job);
     const { module, result } = await job.run();
     return {
@@ -146,6 +147,9 @@ class Loader {
   async getModuleJob(specifier, parentURL) {
     const { url, format } = await this.resolve(specifier, parentURL);
     let job = this.moduleMap.get(url);
+    // CommonJS will set functions for lazy job evaluation.
+    if (typeof job === 'function')
+      this.moduleMap.set(url, job = job());
     if (job !== undefined)
       return job;
 
@@ -169,7 +173,10 @@ class Loader {
       loaderInstance = translators.get(format);
     }
 
-    job = new ModuleJob(this, url, loaderInstance, parentURL === undefined);
+    const inspectBrk = parentURL === undefined &&
+        format === 'module' && getOptionValue('--inspect-brk');
+    job = new ModuleJob(this, url, loaderInstance, parentURL === undefined,
+                        inspectBrk);
     this.moduleMap.set(url, job);
     return job;
   }

lib/internal/modules/esm/module_job.js

@@ -9,7 +9,6 @@ const {
 const { ModuleWrap } = internalBinding('module_wrap');
 
 const { decorateErrorStack } = require('internal/util');
-const { getOptionValue } = require('internal/options');
 const assert = require('internal/assert');
 const resolvedPromise = SafePromise.resolve();
 
@@ -22,9 +21,10 @@ let hasPausedEntry = false;
 class ModuleJob {
   // `loader` is the Loader instance used for loading dependencies.
   // `moduleProvider` is a function
-  constructor(loader, url, moduleProvider, isMain) {
+  constructor(loader, url, moduleProvider, isMain, inspectBrk) {
     this.loader = loader;
     this.isMain = isMain;
+    this.inspectBrk = inspectBrk;
 
     // This is a Promise<{ module, reflect }>, whose fields will be copied
     // onto `this` by `link()` below once it has been resolved.
@@ -83,12 +83,12 @@ class ModuleJob {
     };
     await addJobsToDependencyGraph(this);
     try {
-      if (!hasPausedEntry && this.isMain && getOptionValue('--inspect-brk')) {
+      if (!hasPausedEntry && this.inspectBrk) {
         hasPausedEntry = true;
         const initWrapper = internalBinding('inspector').callAndPauseOnStart;
         initWrapper(this.module.instantiate, this.module);
       } else {
-        this.module.instantiate();
+        this.module.instantiate(true);
       }
     } catch (e) {
       decorateErrorStack(e);

lib/internal/modules/esm/module_map.js

@@ -16,7 +16,8 @@ class ModuleMap extends SafeMap {
   }
   set(url, job) {
     validateString(url, 'url');
-    if (job instanceof ModuleJob !== true) {
+    if (job instanceof ModuleJob !== true &&
+        typeof job !== 'function') {
       throw new ERR_INVALID_ARG_TYPE('job', 'ModuleJob', job);
     }
     debug(`Storing ${url} in ModuleMap`);

lib/internal/modules/esm/translators.js

@@ -23,7 +23,6 @@ const fs = require('fs');
 const { fileURLToPath, URL } = require('url');
 const { debuglog } = require('internal/util/debuglog');
 const { promisify } = require('internal/util');
-const esmLoader = require('internal/process/esm_loader');
 const {
   ERR_INVALID_URL,
   ERR_INVALID_URL_SCHEME,
@@ -69,9 +68,12 @@ function initializeImportMeta(meta, { url }) {
   meta.url = url;
 }
 
+let esmLoader;
 async function importModuleDynamically(specifier, { url }) {
-  const loader = await esmLoader.loaderPromise;
+  if (!esmLoader) {
-  return loader.import(specifier, url);
+    esmLoader = require('internal/process/esm_loader');
+  }
+  return esmLoader.ESMLoader.import(specifier, url);
 }
 
 // Strategy for loading a standard JavaScript module

lib/internal/process/esm_loader.js

@@ -3,12 +3,14 @@
 const {
   ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING,
 } = require('internal/errors').codes;
-
+const assert = require('internal/assert');
 const { Loader } = require('internal/modules/esm/loader');
 const { pathToFileURL } = require('internal/url');
 const {
   getModuleFromWrap,
 } = require('internal/vm/module');
+const { getOptionValue } = require('internal/options');
+const userLoader = getOptionValue('--experimental-loader');
 
 exports.initializeImportMetaObject = function(wrap, meta) {
   const { callbackMap } = internalBinding('module_wrap');
@@ -21,6 +23,7 @@ exports.initializeImportMetaObject = function(wrap, meta) {
 };
 
 exports.importModuleDynamicallyCallback = async function(wrap, specifier) {
+  assert(calledInitialize === true || !userLoader);
   const { callbackMap } = internalBinding('module_wrap');
   if (callbackMap.has(wrap)) {
     const { importModuleDynamically } = callbackMap.get(wrap);
@@ -32,24 +35,31 @@ exports.importModuleDynamicallyCallback = async function(wrap, specifier) {
   throw new ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING();
 };
 
-let loaderResolve;
+let ESMLoader = new Loader();
-exports.loaderPromise = new Promise((resolve) => loaderResolve = resolve);
+exports.ESMLoader = ESMLoader;
 
-exports.ESMLoader = undefined;
+let calledInitialize = false;
-
+exports.initializeLoader = initializeLoader;
-exports.initializeLoader = function(cwd, userLoader) {
+async function initializeLoader() {
-  let ESMLoader = new Loader();
+  assert(calledInitialize === false);
-  const loaderPromise = (async () => {
+  calledInitialize = true;
-    if (userLoader) {
+  if (!userLoader)
-      const hooks = await ESMLoader.import(
+    return;
-        userLoader, pathToFileURL(`${cwd}/`).href);
+  let cwd;
+  try {
+    cwd = process.cwd() + '/';
+  } catch {
+    cwd = 'file:///';
+  }
+  // If --experimental-loader is specified, create a loader with user hooks.
+  // Otherwise create the default loader.
+  const { emitExperimentalWarning } = require('internal/util');
+  emitExperimentalWarning('--experimental-loader');
+  return (async () => {
+    const hooks =
+        await ESMLoader.import(userLoader, pathToFileURL(cwd).href);
     ESMLoader = new Loader();
     ESMLoader.hook(hooks);
-      exports.ESMLoader = ESMLoader;
+    return exports.ESMLoader = ESMLoader;
-    }
-    return ESMLoader;
   })();
-  loaderResolve(loaderPromise);
+}
-
-  exports.ESMLoader = ESMLoader;
-};

lib/internal/process/execution.js

@@ -39,7 +39,7 @@ function evalModule(source, print) {
   const { log, error } = require('internal/console/global');
   const { decorateErrorStack } = require('internal/util');
   const asyncESM = require('internal/process/esm_loader');
-  asyncESM.loaderPromise.then(async (loader) => {
+  Promise.resolve(asyncESM.ESMLoader).then(async (loader) => {
     const { result } = await loader.eval(source);
     if (print) {
       log(result);

lib/repl.js

@@ -99,6 +99,7 @@ const {
 } = internalBinding('contextify');
 
 const history = require('internal/repl/history');
+const { setImmediate } = require('timers');
 
 // Lazy-loaded.
 let processTopLevelAwait;
@@ -338,11 +339,9 @@ function REPLServer(prompt,
         script = vm.createScript(code, {
           filename: file,
           displayErrors: true,
-          importModuleDynamically: experimentalModules ?
+          importModuleDynamically: experimentalModules ? async (specifier) => {
-            async (specifier) => {
+            return asyncESM.ESMLoader.import(specifier, pwd);
-              return (await asyncESM.loaderPromise).import(specifier, pwd);
+          } : undefined
-            } :
-            undefined
         });
       } catch (e) {
         debug('parse error %j', code, e);
@@ -1080,7 +1079,11 @@ function complete(line, callback) {
     // All this is only profitable if the nested REPL does not have a
     // bufferedCommand.
     if (!magic[kBufferedCommandSymbol]) {
-      magic._domain.on('error', (err) => { throw err; });
+      magic._domain.on('error', (err) => {
+        setImmediate(() => {
+          throw err;
+        });
+      });
       return magic.complete(line, callback);
     }
   }

test/es-module/test-esm-loader-modulemap.js

@@ -1,5 +1,5 @@
 'use strict';
-// Flags: --expose-internals
+// Flags: --expose-internals --experimental-modules
 
 // This test ensures that the type checking of ModuleMap throws
 // errors appropriately

test/es-module/test-esm-no-extension.js

@@ -15,11 +15,6 @@ const child = spawn(process.execPath, [
   entry
 ]);
 
-let stderr = '';
-child.stderr.setEncoding('utf8');
-child.stderr.on('data', (data) => {
-  stderr += data;
-});
 let stdout = '';
 child.stdout.setEncoding('utf8');
 child.stdout.on('data', (data) => {
@@ -29,6 +24,4 @@ child.on('close', common.mustCall((code, signal) => {
   assert.strictEqual(code, 0);
   assert.strictEqual(signal, null);
   assert.strictEqual(stdout, 'executed\n');
-  assert.strictEqual(stderr, `(node:${child.pid}) ` +
-      'ExperimentalWarning: The ESM module loader is experimental.\n');
 }));

test/parallel/test-bootstrap-modules.js

@@ -92,7 +92,6 @@ if (common.isMainThread) {
   expectedModules.add('NativeModule internal/streams/state');
   expectedModules.add('NativeModule internal/worker');
   expectedModules.add('NativeModule internal/worker/io');
-  expectedModules.add('NativeModule module');
   expectedModules.add('NativeModule stream');
   expectedModules.add('NativeModule worker_threads');
 }

test/parallel/test-internal-module-map-asserts.js

@@ -12,7 +12,7 @@ const ModuleMap = require('internal/modules/esm/module_map');
     code: 'ERR_INVALID_ARG_TYPE',
     type: TypeError,
     message: /^The "url" argument must be of type string/
-  }, 15);
+  }, 12);
 
   const moduleMap = new ModuleMap();
 
@@ -21,7 +21,7 @@ const ModuleMap = require('internal/modules/esm/module_map');
   // but I think it's useless, and was not simple to mock...
   const job = undefined;
 
-  [{}, [], true, 1, () => {}].forEach((value) => {
+  [{}, [], true, 1].forEach((value) => {
     assert.throws(() => moduleMap.get(value), errorReg);
     assert.throws(() => moduleMap.has(value), errorReg);
     assert.throws(() => moduleMap.set(value, job), errorReg);
@@ -34,11 +34,11 @@ const ModuleMap = require('internal/modules/esm/module_map');
     code: 'ERR_INVALID_ARG_TYPE',
     type: TypeError,
     message: /^The "job" argument must be of type ModuleJob/
-  }, 5);
+  }, 4);
 
   const moduleMap = new ModuleMap();
 
-  [{}, [], true, 1, () => {}].forEach((value) => {
+  [{}, [], true, 1].forEach((value) => {
     assert.throws(() => moduleMap.set('', value), errorReg);
   });
 }