lib: improve module loading performance

This commit improves module loading performance by at least ~25-35%
in the module-loader benchmarks.

Some optimization strategies include:
* Try-finally/try-catch isolation
* Replacing regular expressions with manual parsing
* Avoiding unnecessary string and array creation
* Avoiding constant recompilation of anonymous functions and
function definitions within functions

PR-URL: https://github.com/nodejs/node/pull/5172
Reviewed-By: James M Snell <jasnell@gmail.com>

lib/fs.js

@@ -450,6 +450,42 @@ function tryToString(buf, encoding, callback) {
   callback(e, buf);
 }
 
+function tryStatSync(fd, isUserFd) {
+  var threw = true;
+  var st;
+  try {
+    st = fs.fstatSync(fd);
+    threw = false;
+  } finally {
+    if (threw && !isUserFd) fs.closeSync(fd);
+  }
+  return st;
+}
+
+function tryCreateBuffer(size, fd, isUserFd) {
+  var threw = true;
+  var buffer;
+  try {
+    buffer = Buffer.allocUnsafe(size);
+    threw = false;
+  } finally {
+    if (threw && !isUserFd) fs.closeSync(fd);
+  }
+  return buffer;
+}
+
+function tryReadSync(fd, isUserFd, buffer, pos, len) {
+  var threw = true;
+  var bytesRead;
+  try {
+    bytesRead = fs.readSync(fd, buffer, pos, len);
+    threw = false;
+  } finally {
+    if (threw && !isUserFd) fs.closeSync(fd);
+  }
+  return bytesRead;
+}
+
 fs.readFileSync = function(path, options) {
   if (!options) {
     options = { encoding: null, flag: 'r' };
@@ -466,17 +502,8 @@ fs.readFileSync = function(path, options) {
   var isUserFd = isFd(path); // file descriptor ownership
   var fd = isUserFd ? path : fs.openSync(path, flag, 0o666);
 
-  var st;
+  var st = tryStatSync(fd, isUserFd);
-  var size;
+  var size = st.isFile() ? st.size : 0;
-  var threw = true;
-  try {
-    st = fs.fstatSync(fd);
-    size = st.isFile() ? st.size : 0;
-    threw = false;
-  } finally {
-    if (threw && !isUserFd) fs.closeSync(fd);
-  }
-
   var pos = 0;
   var buffer; // single buffer with file data
   var buffers; // list for when size is unknown
@@ -484,39 +511,27 @@ fs.readFileSync = function(path, options) {
   if (size === 0) {
     buffers = [];
   } else {
-    threw = true;
+    buffer = tryCreateBuffer(size, fd, isUserFd);
-    try {
-      buffer = Buffer.allocUnsafe(size);
-      threw = false;
-    } finally {
-      if (threw && !isUserFd) fs.closeSync(fd);
-    }
   }
 
-  var done = false;
   var bytesRead;
 
-  while (!done) {
+  if (size !== 0) {
-    threw = true;
+    do {
-    try {
+      bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
-      if (size !== 0) {
+      pos += bytesRead;
-        bytesRead = fs.readSync(fd, buffer, pos, size - pos);
+    } while (bytesRead !== 0 && pos < size);
-      } else {
+  } else {
-        // the kernel lies about many files.
+    do {
-        // Go ahead and try to read some bytes.
+      // the kernel lies about many files.
-        buffer = Buffer.allocUnsafe(8192);
+      // Go ahead and try to read some bytes.
-        bytesRead = fs.readSync(fd, buffer, 0, 8192);
+      buffer = Buffer.allocUnsafe(8192);
-        if (bytesRead) {
+      bytesRead = tryReadSync(fd, isUserFd, buffer, 0, 8192);
-          buffers.push(buffer.slice(0, bytesRead));
+      if (bytesRead !== 0) {
-        }
+        buffers.push(buffer.slice(0, bytesRead));
       }
-      threw = false;
+      pos += bytesRead;
-    } finally {
+    } while (bytesRead !== 0);
-      if (threw && !isUserFd) fs.closeSync(fd);
-    }
-
-    pos += bytesRead;
-    done = (bytesRead === 0) || (size !== 0 && pos >= size);
   }
 
   if (!isUserFd)

lib/internal/bootstrap_node.js

@@ -284,37 +284,45 @@
     };
   }
 
-  function evalScript(name) {
+  function tryGetCwd(path) {
-    var Module = NativeModule.require('module');
+    var threw = true;
-    var path = NativeModule.require('path');
+    var cwd;
-
     try {
-      var cwd = process.cwd();
+      cwd = process.cwd();
-    } catch (e) {
+      threw = false;
-      // getcwd(3) can fail if the current working directory has been deleted.
+    } finally {
-      // Fall back to the directory name of the (absolute) executable path.
+      if (threw) {
-      // It's not really correct but what are the alternatives?
+        // getcwd(3) can fail if the current working directory has been deleted.
-      cwd = path.dirname(process.execPath);
+        // Fall back to the directory name of the (absolute) executable path.
+        // It's not really correct but what are the alternatives?
+        return path.dirname(process.execPath);
+      }
     }
+    return cwd;
+  }
 
-    var module = new Module(name);
+  function evalScript(name) {
+    const Module = NativeModule.require('module');
+    const path = NativeModule.require('path');
+    const cwd = tryGetCwd(path);
+
+    const module = new Module(name);
     module.filename = path.join(cwd, name);
     module.paths = Module._nodeModulePaths(cwd);
-    var script = process._eval;
+    const body = process._eval;
-    var body = script;
+    const script = `global.__filename = ${JSON.stringify(name)};\n` +
-    script = `global.__filename = ${JSON.stringify(name)};\n` +
+                   'global.exports = exports;\n' +
-             'global.exports = exports;\n' +
+                   'global.module = module;\n' +
-             'global.module = module;\n' +
+                   'global.__dirname = __dirname;\n' +
-             'global.__dirname = __dirname;\n' +
+                   'global.require = require;\n' +
-             'global.require = require;\n' +
+                   'return require("vm").runInThisContext(' +
-             'return require("vm").runInThisContext(' +
+                   `${JSON.stringify(body)}, { filename: ` +
-             `${JSON.stringify(body)}, { filename: ` +
+                   `${JSON.stringify(name)}, displayErrors: true });\n`;
-             `${JSON.stringify(name)}, displayErrors: true });\n`;
     // Defer evaluation for a tick.  This is a workaround for deferred
     // events not firing when evaluating scripts from the command line,
     // see https://github.com/nodejs/node/issues/1600.
     process.nextTick(function() {
-      var result = module._compile(script, `${name}-wrapper`);
+      const result = module._compile(script, `${name}-wrapper`);
       if (process._print_eval) console.log(result);
     });
   }

lib/internal/module.js

@@ -19,9 +19,11 @@ function makeRequireFunction() {
     }
   }
 
-  require.resolve = function(request) {
+  function resolve(request) {
     return Module._resolveFilename(request, self);
-  };
+  }
+
+  require.resolve = resolve;
 
   require.main = process.mainModule;
 

lib/module.js

@@ -11,10 +11,6 @@ const path = require('path');
 const internalModuleReadFile = process.binding('fs').internalModuleReadFile;
 const internalModuleStat = process.binding('fs').internalModuleStat;
 
-const splitRe = process.platform === 'win32' ? /[\/\\]/ : /\//;
-const isIndexRe = /^index\.\w+?$/;
-const shebangRe = /^\#\!.*/;
-
 // If obj.hasOwnProperty has been overridden, then calling
 // obj.hasOwnProperty(prop) will break.
 // See: https://github.com/joyent/node/issues/1707
@@ -84,8 +80,8 @@ function readPackage(requestPath) {
     return packageMainCache[requestPath];
   }
 
-  var jsonPath = path.resolve(requestPath, 'package.json');
+  const jsonPath = path.resolve(requestPath, 'package.json');
-  var json = internalModuleReadFile(path._makeLong(jsonPath));
+  const json = internalModuleReadFile(path._makeLong(jsonPath));
 
   if (json === undefined) {
     return false;
@@ -107,7 +103,8 @@ function tryPackage(requestPath, exts) {
   if (!pkg) return false;
 
   var filename = path.resolve(requestPath, pkg);
-  return tryFile(filename) || tryExtensions(filename, exts) ||
+  return tryFile(filename) ||
+         tryExtensions(filename, exts) ||
          tryExtensions(path.resolve(filename, 'index'), exts);
 }
 
@@ -128,8 +125,8 @@ function toRealPath(requestPath) {
 
 // given a path check a the file exists with any of the set extensions
 function tryExtensions(p, exts) {
-  for (var i = 0, EL = exts.length; i < EL; i++) {
+  for (var i = 0; i < exts.length; i++) {
-    var filename = tryFile(p + exts[i]);
+    const filename = tryFile(p + exts[i]);
 
     if (filename) {
       return filename;
@@ -142,21 +139,25 @@ var warned = false;
 Module._findPath = function(request, paths) {
   if (path.isAbsolute(request)) {
     paths = [''];
+  } else if (!paths || paths.length === 0) {
+    return false;
   }
 
-  var cacheKey = JSON.stringify({request: request, paths: paths});
+  const cacheKey = JSON.stringify({request: request, paths: paths});
   if (Module._pathCache[cacheKey]) {
     return Module._pathCache[cacheKey];
   }
 
-  const exts = Object.keys(Module._extensions);
+  var exts;
-  const trailingSlash = request.slice(-1) === '/';
+  const trailingSlash = request.length > 0 &&
+                        request.charCodeAt(request.length - 1) === 47/*/*/;
 
   // For each path
-  for (var i = 0, PL = paths.length; i < PL; i++) {
+  for (var i = 0; i < paths.length; i++) {
     // Don't search further if path doesn't exist
-    if (paths[i] && stat(paths[i]) < 1) continue;
+    const curPath = paths[i];
-    var basePath = path.resolve(paths[i], request);
+    if (curPath && stat(curPath) < 1) continue;
+    var basePath = path.resolve(curPath, request);
     var filename;
 
     if (!trailingSlash) {
@@ -164,11 +165,15 @@ Module._findPath = function(request, paths) {
       if (rc === 0) {  // File.
         filename = toRealPath(basePath);
       } else if (rc === 1) {  // Directory.
+        if (exts === undefined)
+          exts = Object.keys(Module._extensions);
         filename = tryPackage(basePath, exts);
       }
 
       if (!filename) {
         // try it with each of the extensions
+        if (exts === undefined)
+          exts = Object.keys(Module._extensions);
         filename = tryExtensions(basePath, exts);
       }
     }
@@ -179,6 +184,8 @@ Module._findPath = function(request, paths) {
 
     if (!filename) {
       // try it with each of the extensions at "index"
+      if (exts === undefined)
+        exts = Object.keys(Module._extensions);
       filename = tryExtensions(path.resolve(basePath, 'index'), exts);
     }
 
@@ -198,48 +205,105 @@ Module._findPath = function(request, paths) {
   return false;
 };
 
-// 'from' is the __dirname of the module.
+// 'node_modules' character codes reversed
-Module._nodeModulePaths = function(from) {
+var nmChars = [ 115, 101, 108, 117, 100, 111, 109, 95, 101, 100, 111, 110 ];
-  // guarantee that 'from' is absolute.
+var nmLen = nmChars.length;
-  from = path.resolve(from);
+if (process.platform === 'win32') {
+  // 'from' is the __dirname of the module.
+  Module._nodeModulePaths = function(from) {
+    // guarantee that 'from' is absolute.
+    from = path.resolve(from);
 
-  // note: this approach *only* works when the path is guaranteed
+    // note: this approach *only* works when the path is guaranteed
-  // to be absolute.  Doing a fully-edge-case-correct path.split
+    // to be absolute.  Doing a fully-edge-case-correct path.split
-  // that works on both Windows and Posix is non-trivial.
+    // that works on both Windows and Posix is non-trivial.
-  var paths = [];
+    const paths = [];
-  var parts = from.split(splitRe);
+    var p = 0;
+    var last = from.length;
+    for (var i = from.length - 1; i >= 0; --i) {
+      const code = from.charCodeAt(i);
+      if (code === 92/*\*/ || code === 47/*/*/) {
+        if (p !== nmLen)
+          paths.push(from.slice(0, last) + '\\node_modules');
+        last = i;
+        p = 0;
+      } else if (p !== -1 && p < nmLen) {
+        if (nmChars[p] === code) {
+          ++p;
+        } else {
+          p = -1;
+        }
+      }
+    }
 
-  for (var tip = parts.length - 1; tip >= 0; tip--) {
+    return paths;
-    // don't search in .../node_modules/node_modules
+  };
-    if (parts[tip] === 'node_modules') continue;
+} else { // posix
-    var dir = parts.slice(0, tip + 1).concat('node_modules').join(path.sep);
+  // 'from' is the __dirname of the module.
-    paths.push(dir);
+  Module._nodeModulePaths = function(from) {
-  }
+    // guarantee that 'from' is absolute.
+    from = path.resolve(from);
+    // Return early not only to avoid unnecessary work, but to *avoid* returning
+    // an array of two items for a root: [ '//node_modules', '/node_modules' ]
+    if (from === '/')
+      return ['/node_modules'];
 
-  return paths;
+    // note: this approach *only* works when the path is guaranteed
-};
+    // to be absolute.  Doing a fully-edge-case-correct path.split
+    // that works on both Windows and Posix is non-trivial.
+    const paths = [];
+    var p = 0;
+    var last = from.length;
+    for (var i = from.length - 1; i >= 0; --i) {
+      const code = from.charCodeAt(i);
+      if (code === 47/*/*/) {
+        if (p !== nmLen)
+          paths.push(from.slice(0, last) + '/node_modules');
+        last = i;
+        p = 0;
+      } else if (p !== -1 && p < nmLen) {
+        if (nmChars[p] === code) {
+          ++p;
+        } else {
+          p = -1;
+        }
+      }
+    }
+
+    return paths;
+  };
+}
 
 
+// 'index.' character codes
+var indexChars = [ 105, 110, 100, 101, 120, 46 ];
+var indexLen = indexChars.length;
 Module._resolveLookupPaths = function(request, parent) {
   if (NativeModule.nonInternalExists(request)) {
     return [request, []];
   }
 
-  var start = request.substring(0, 2);
+  var reqLen = request.length;
-  if (start !== './' && start !== '..') {
+  // Check for relative path
+  if (reqLen < 2 ||
+      request.charCodeAt(0) !== 46/*.*/ ||
+      (request.charCodeAt(1) !== 46/*.*/ &&
+       request.charCodeAt(1) !== 47/*/*/)) {
     var paths = modulePaths;
     if (parent) {
-      if (!parent.paths) parent.paths = [];
+      if (!parent.paths)
-      paths = parent.paths.concat(paths);
+        paths = parent.paths = [];
+      else
+        paths = parent.paths.concat(paths);
     }
 
     // Maintain backwards compat with certain broken uses of require('.')
     // by putting the module's directory in front of the lookup paths.
     if (request === '.') {
       if (parent && parent.filename) {
-        paths.splice(0, 0, path.dirname(parent.filename));
+        paths.unshift(path.dirname(parent.filename));
       } else {
-        paths.splice(0, 0, path.resolve(request));
+        paths.unshift(path.resolve(request));
       }
     }
 
@@ -257,8 +321,39 @@ Module._resolveLookupPaths = function(request, parent) {
   // Is the parent an index module?
   // We can assume the parent has a valid extension,
   // as it already has been accepted as a module.
-  var isIndex = isIndexRe.test(path.basename(parent.filename));
+  const base = path.basename(parent.filename);
-  var parentIdPath = isIndex ? parent.id : path.dirname(parent.id);
+  var parentIdPath;
+  if (base.length > indexLen) {
+    var i = 0;
+    for (; i < indexLen; ++i) {
+      if (indexChars[i] !== base.charCodeAt(i))
+        break;
+    }
+    if (i === indexLen) {
+      // We matched 'index.', let's validate the rest
+      for (; i < base.length; ++i) {
+        const code = base.charCodeAt(i);
+        if (code !== 95/*_*/ &&
+            (code < 48/*0*/ || code > 57/*9*/) &&
+            (code < 65/*A*/ || code > 90/*Z*/) &&
+            (code < 97/*a*/ || code > 122/*z*/))
+          break;
+      }
+      if (i === base.length) {
+        // Is an index module
+        parentIdPath = parent.id;
+      } else {
+        // Not an index module
+        parentIdPath = path.dirname(parent.id);
+      }
+    } else {
+      // Not an index module
+      parentIdPath = path.dirname(parent.id);
+    }
+  } else {
+    // Not an index module
+    parentIdPath = path.dirname(parent.id);
+  }
   var id = path.resolve(parentIdPath, request);
 
   // make sure require('./path') and require('path') get distinct ids, even
@@ -307,20 +402,23 @@ Module._load = function(request, parent, isMain) {
 
   Module._cache[filename] = module;
 
-  var hadException = true;
+  tryModuleLoad(module, filename);
-
-  try {
-    module.load(filename);
-    hadException = false;
-  } finally {
-    if (hadException) {
-      delete Module._cache[filename];
-    }
-  }
 
   return module.exports;
 };
 
+function tryModuleLoad(module, filename) {
+  var threw = true;
+  try {
+    module.load(filename);
+    threw = false;
+  } finally {
+    if (threw) {
+      delete Module._cache[filename];
+    }
+  }
+}
+
 Module._resolveFilename = function(request, parent) {
   if (NativeModule.nonInternalExists(request)) {
     return request;
@@ -377,8 +475,33 @@ var resolvedArgv;
 // the file.
 // Returns exception, if any.
 Module.prototype._compile = function(content, filename) {
-  // remove shebang
+  // Remove shebang
-  content = content.replace(shebangRe, '');
+  var contLen = content.length;
+  if (contLen >= 2) {
+    if (content.charCodeAt(0) === 35/*#*/ &&
+        content.charCodeAt(1) === 33/*!*/) {
+      if (contLen === 2) {
+        // Exact match
+        content = '';
+      } else {
+        // Find end of shebang line and slice it off
+        var i = 2;
+        for (; i < contLen; ++i) {
+          var code = content.charCodeAt(i);
+          if (code === 10/*\n*/ || code === 13/*\r*/)
+            break;
+        }
+        if (i === contLen)
+          content = '';
+        else {
+          // Note that this actually includes the newline character(s) in the
+          // new output. This duplicates the behavior of the regular expression
+          // that was previously used to replace the shebang line
+          content = content.slice(i);
+        }
+      }
+    }
+  }
 
   // create wrapper function
   var wrapper = Module.wrap(content);
@@ -408,12 +531,12 @@ Module.prototype._compile = function(content, filename) {
       global.v8debug.Debug.setBreakPoint(compiledWrapper, 0, 0);
     }
   }
-  const dirname = path.dirname(filename);
+  var dirname = path.dirname(filename);
-  const require = internalModule.makeRequireFunction.call(this);
+  var require = internalModule.makeRequireFunction.call(this);
-  const args = [this.exports, require, this, filename, dirname];
+  var args = [this.exports, require, this, filename, dirname];
-  const depth = internalModule.requireDepth;
+  var depth = internalModule.requireDepth;
   if (depth === 0) stat.cache = new Map();
-  const result = compiledWrapper.apply(this.exports, args);
+  var result = compiledWrapper.apply(this.exports, args);
   if (depth === 0) stat.cache = null;
   return result;
 };

test/message/core_line_numbers.out

@@ -9,7 +9,7 @@ RangeError: Invalid input
     at Module._compile (module.js:*:*)
     at Object.Module._extensions..js (module.js:*:*)
     at Module.load (module.js:*:*)
+    at tryModuleLoad (module.js:*:*)
     at Function.Module._load (module.js:*:*)
     at Function.Module.runMain (module.js:*:*)
     at startup (node.js:*:*)
-    at node.js:*:*

test/message/error_exit.out

@@ -8,6 +8,7 @@ AssertionError: 1 == 2
     at Module._compile (module.js:*:*)
     at Object.Module._extensions..js (module.js:*:*)
     at Module.load (module.js:*:*)
+    at tryModuleLoad (module.js:*:*)
     at Function.Module._load (module.js:*:*)
     at Function.Module.runMain (module.js:*:*)
     at startup (node.js:*:*)

test/message/undefined_reference_in_new_context.out

@@ -11,6 +11,6 @@ ReferenceError: foo is not defined
     at Module._compile (module.js:*)
     at *..js (module.js:*)
     at Module.load (module.js:*)
+    at tryModuleLoad (module.js:*:*)
     at Function.Module._load (module.js:*:*)
     at Function.Module.runMain (module.js:*:*)
-    at startup (node.js:*:*)

test/message/vm_display_runtime_error.out

@@ -10,7 +10,7 @@ Error: boo!
     at Module._compile (module.js:*)
     at Object.Module._extensions..js (module.js:*)
     at Module.load (module.js:*)
+    at tryModuleLoad (module.js:*:*)
     at Function.Module._load (module.js:*)
     at Function.Module.runMain (module.js:*)
     at startup (node.js:*)
-    at node.js:*

test/message/vm_display_syntax_error.out

@@ -9,6 +9,7 @@ SyntaxError: Unexpected number
     at Module._compile (module.js:*)
     at Object.Module._extensions..js (module.js:*)
     at Module.load (module.js:*)
+    at tryModuleLoad (module.js:*:*)
     at Function.Module._load (module.js:*)
     at Function.Module.runMain (module.js:*)
     at startup (node.js:*)
@@ -22,6 +23,7 @@ SyntaxError: Unexpected number
     at Module._compile (module.js:*)
     at Object.Module._extensions..js (module.js:*)
     at Module.load (module.js:*)
+    at tryModuleLoad (module.js:*:*)
     at Function.Module._load (module.js:*)
     at Function.Module.runMain (module.js:*)
     at startup (node.js:*)

test/message/vm_dont_display_runtime_error.out

@@ -10,7 +10,7 @@ Error: boo!
     at Module._compile (module.js:*)
     at Object.Module._extensions..js (module.js:*)
     at Module.load (module.js:*)
+    at tryModuleLoad (module.js:*:*)
     at Function.Module._load (module.js:*)
     at Function.Module.runMain (module.js:*)
     at startup (node.js:*)
-    at node.js:*

test/message/vm_dont_display_syntax_error.out

@@ -9,6 +9,7 @@ SyntaxError: Unexpected number
     at Module._compile (module.js:*)
     at Object.Module._extensions..js (module.js:*)
     at Module.load (module.js:*)
+    at tryModuleLoad (module.js:*:*)
     at Function.Module._load (module.js:*)
     at Function.Module.runMain (module.js:*)
     at startup (node.js:*)