From 83f8d98806c3a28b611c848256350aec32211b83 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Thu, 7 Jan 2016 22:36:30 +0100 Subject: [PATCH] module: cache stat() results more aggressively Reduce the number of stat() system calls that require() makes by caching the results more aggressively. To avoid unbounded growth without implementing a LRU cache, scope the cache to the lifetime of the first call to require(). Recursive calls (i.e. require() calls in the included code) transparently profit from the cache. The benchmarked application is the loopback-sample-app[0] and it sees the number of stat calls at start-up go down by 40%, from 4736 to 2810. [0] https://github.com/strongloop/loopback-sample-app PR-URL: https://github.com/nodejs/node/pull/4575 Reviewed-By: Colin Ihrig Reviewed-By: James M Snell --- lib/internal/module.js | 11 +++++++-- lib/module.js | 26 ++++++++++++++++++---- test/fixtures/module-require-depth/one.js | 9 ++++++++ test/fixtures/module-require-depth/two.js | 9 ++++++++ test/parallel/test-module-require-depth.js | 13 +++++++++++ 5 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/module-require-depth/one.js create mode 100644 test/fixtures/module-require-depth/two.js create mode 100644 test/parallel/test-module-require-depth.js diff --git a/lib/internal/module.js b/lib/internal/module.js index ef55aa64bd5..29f88999dbf 100644 --- a/lib/internal/module.js +++ b/lib/internal/module.js @@ -1,6 +1,8 @@ 'use strict'; -module.exports = { makeRequireFunction, stripBOM }; +exports = module.exports = { makeRequireFunction, stripBOM }; + +exports.requireDepth = 0; // Invoke with makeRequireFunction.call(module) where |module| is the // Module object to use as the context for the require() function. @@ -9,7 +11,12 @@ function makeRequireFunction() { const self = this; function require(path) { - return self.require(path); + try { + exports.requireDepth += 1; + return self.require(path); + } finally { + exports.requireDepth -= 1; + } } require.resolve = function(request) { diff --git a/lib/module.js b/lib/module.js index e403c1f73bc..b51772ad59d 100644 --- a/lib/module.js +++ b/lib/module.js @@ -33,6 +33,20 @@ function tryWrapper(wrapper, opts) { } +function stat(filename) { + filename = path._makeLong(filename); + const cache = stat.cache; + if (cache !== null) { + const result = cache.get(filename); + if (result !== undefined) return result; + } + const result = internalModuleStat(filename); + if (cache !== null) cache.set(filename, result); + return result; +} +stat.cache = null; + + function Module(id, parent) { this.id = id; this.exports = {}; @@ -114,7 +128,7 @@ Module._realpathCache = {}; // check if the file exists and is not a directory function tryFile(requestPath) { - const rc = internalModuleStat(path._makeLong(requestPath)); + const rc = stat(requestPath); return rc === 0 && toRealPath(requestPath); } @@ -151,12 +165,12 @@ Module._findPath = function(request, paths) { // For each path for (var i = 0, PL = paths.length; i < PL; i++) { // Don't search further if path doesn't exist - if (paths[i] && internalModuleStat(path._makeLong(paths[i])) < 1) continue; + if (paths[i] && stat(paths[i]) < 1) continue; var basePath = path.resolve(paths[i], request); var filename; if (!trailingSlash) { - const rc = internalModuleStat(path._makeLong(basePath)); + const rc = stat(basePath); if (rc === 0) { // File. filename = toRealPath(basePath); } else if (rc === 1) { // Directory. @@ -404,7 +418,11 @@ Module.prototype._compile = function(content, filename) { const dirname = path.dirname(filename); const require = internalModule.makeRequireFunction.call(this); const args = [this.exports, require, this, filename, dirname]; - return compiledWrapper.apply(this.exports, args); + const depth = internalModule.requireDepth; + if (depth === 0) stat.cache = new Map(); + const result = compiledWrapper.apply(this.exports, args); + if (depth === 0) stat.cache = null; + return result; }; diff --git a/test/fixtures/module-require-depth/one.js b/test/fixtures/module-require-depth/one.js new file mode 100644 index 00000000000..5927908b754 --- /dev/null +++ b/test/fixtures/module-require-depth/one.js @@ -0,0 +1,9 @@ +// Flags: --expose_internals +'use strict'; +const assert = require('assert'); +const internalModule = require('internal/module'); + +exports.requireDepth = internalModule.requireDepth; +assert.strictEqual(internalModule.requireDepth, 1); +assert.deepStrictEqual(require('./two'), { requireDepth: 2 }); +assert.strictEqual(internalModule.requireDepth, 1); diff --git a/test/fixtures/module-require-depth/two.js b/test/fixtures/module-require-depth/two.js new file mode 100644 index 00000000000..aea49947d11 --- /dev/null +++ b/test/fixtures/module-require-depth/two.js @@ -0,0 +1,9 @@ +// Flags: --expose_internals +'use strict'; +const assert = require('assert'); +const internalModule = require('internal/module'); + +exports.requireDepth = internalModule.requireDepth; +assert.strictEqual(internalModule.requireDepth, 2); +assert.deepStrictEqual(require('./one'), { requireDepth: 1 }); +assert.strictEqual(internalModule.requireDepth, 2); diff --git a/test/parallel/test-module-require-depth.js b/test/parallel/test-module-require-depth.js new file mode 100644 index 00000000000..4d2ddac151b --- /dev/null +++ b/test/parallel/test-module-require-depth.js @@ -0,0 +1,13 @@ +// Flags: --expose_internals +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const internalModule = require('internal/module'); + +// Module one loads two too so the expected depth for two is, well, two. +assert.strictEqual(internalModule.requireDepth, 0); +const one = require(common.fixturesDir + '/module-require-depth/one'); +const two = require(common.fixturesDir + '/module-require-depth/two'); +assert.deepStrictEqual(one, { requireDepth: 1 }); +assert.deepStrictEqual(two, { requireDepth: 2 }); +assert.strictEqual(internalModule.requireDepth, 0);