Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

module: improve require() performance #10789

Merged
merged 10 commits into from
Mar 11, 2017
Merged

module: improve require() performance #10789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
190dc69
benchmark: add parameter for module benchmark
mscdex Jan 13, 2017
21b2440
fs: avoid recompilation of closure
mscdex Jan 13, 2017
34c9fc2
fs: avoid multiple conversions to string
mscdex Jan 13, 2017
1c3df96
fs: replace regexp with function
mscdex Jan 13, 2017
a851b86
lib: remove sources of permanent deopts
mscdex Jan 13, 2017
298a40e
module: use "clean" objects
mscdex Jan 13, 2017
403b89e
module: avoid hasOwnProperty()
mscdex Jan 13, 2017
28dc848
lib: improve method of function calling
mscdex Jan 13, 2017
e32425b
module: avoid JSON.stringify() for cache key
mscdex Jan 13, 2017
c672077
lib: simplify Module._resolveLookupPaths
mscdex Jan 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions benchmark/module/module-loader.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ var benchmarkDirectory = path.join(tmpDirectory, 'nodejs-benchmark-module');

var bench = common.createBenchmark(main, {
thousands: [50],
fullPath: ['true', 'false']
fullPath: ['true', 'false'],
useCache: ['true', 'false']
});

function main(conf) {
Expand All @@ -31,22 +32,34 @@ function main(conf) {
}

if (conf.fullPath === 'true')
measureFull(n);
measureFull(n, conf.useCache === 'true');
else
measureDir(n);
measureDir(n, conf.useCache === 'true');
}

function measureFull(n) {
function measureFull(n, useCache) {
var i;
if (useCache) {
for (i = 0; i <= n; i++) {
require(benchmarkDirectory + i + '/index.js');
}
}
bench.start();
for (var i = 0; i <= n; i++) {
for (i = 0; i <= n; i++) {
require(benchmarkDirectory + i + '/index.js');
}
bench.end(n / 1e3);
}

function measureDir(n) {
function measureDir(n, useCache) {
var i;
if (useCache) {
for (i = 0; i <= n; i++) {
require(benchmarkDirectory + i);
}
}
bench.start();
for (var i = 0; i <= n; i++) {
for (i = 0; i <= n; i++) {
require(benchmarkDirectory + i);
}
bench.end(n / 1e3);
Expand Down
161 changes: 99 additions & 62 deletions lib/fs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1478,12 +1478,6 @@ fs.unwatchFile = function(filename, listener) {
};


// Regexp that finds the next portion of a (partial) path
// result is [base_with_slash, base], e.g. ['somedir/', 'somedir']
const nextPartRe = isWindows ?
/(.*?)(?:[/\\]+|$)/g :
/(.*?)(?:[/]+|$)/g;

// Regex to find the device root, including trailing slash. E.g. 'c:\\'.
const splitRootRe = isWindows ?
/^(?:[a-zA-Z]:|[\\/]{2}[^\\/]+[\\/][^\\/]+)?[\\/]*/ :
Expand All @@ -1500,24 +1494,39 @@ function encodeRealpathResult(result, options) {
}
}

// Finds the next portion of a (partial) path, up to the next path delimiter
var nextPart;
if (isWindows) {
nextPart = function nextPart(p, i) {
for (; i < p.length; ++i) {
const ch = p.charCodeAt(i);
if (ch === 92/*'\'*/ || ch === 47/*'/'*/)
return i;
}
return -1;
};
} else {
nextPart = function nextPart(p, i) { return p.indexOf('/', i); };
}

fs.realpathSync = function realpathSync(p, options) {
options = getOptions(options, {});
handleError((p = getPathFromURL(p)));
if (typeof p !== 'string')
p += '';
nullCheck(p);

p = p.toString('utf8');
p = pathModule.resolve(p);

const seenLinks = {};
const knownHard = {};
const cache = options[internalFS.realpathCacheKey];
const original = p;

const maybeCachedResult = cache && cache.get(p);
if (maybeCachedResult) {
return maybeCachedResult;
}

const seenLinks = {};
const knownHard = {};
const original = p;

// current character position in p
var pos;
// the partial path so far, including a trailing slash if any
Expand All @@ -1527,42 +1536,43 @@ fs.realpathSync = function realpathSync(p, options) {
// the partial path scanned in the previous round, with slash
var previous;

start();

function start() {
// Skip over roots
var m = splitRootRe.exec(p);
pos = m[0].length;
current = m[0];
base = m[0];
previous = '';
// Skip over roots
var m = splitRootRe.exec(p);
pos = m[0].length;
current = m[0];
base = m[0];

// On windows, check that the root exists. On unix there is no need.
if (isWindows && !knownHard[base]) {
fs.lstatSync(base);
knownHard[base] = true;
}
// On windows, check that the root exists. On unix there is no need.
if (isWindows && !knownHard[base]) {
fs.lstatSync(base);
knownHard[base] = true;
}

// walk down the path, swapping out linked pathparts for their real
// values
// NB: p.length changes.
while (pos < p.length) {
// find the next part
nextPartRe.lastIndex = pos;
var result = nextPartRe.exec(p);
var result = nextPart(p, pos);
previous = current;
current += result[0];
base = previous + result[1];
pos = nextPartRe.lastIndex;
if (result === -1) {
var last = p.slice(pos);
current += last;
base = previous + last;
pos = p.length;
} else {
current += p.slice(pos, result + 1);
base = previous + p.slice(pos, result);
pos = result + 1;
}

// continue if not a symlink
if (knownHard[base] || (cache && cache.get(base) === base)) {
continue;
}

var resolvedLink;
const maybeCachedResolved = cache && cache.get(base);
var maybeCachedResolved = cache && cache.get(base);
if (maybeCachedResolved) {
resolvedLink = maybeCachedResolved;
} else {
Expand All @@ -1575,8 +1585,8 @@ fs.realpathSync = function realpathSync(p, options) {

// read the link if it wasn't read before
// dev/ino always return 0 on windows, so skip the check.
let linkTarget = null;
let id;
var linkTarget = null;
var id;
if (!isWindows) {
id = `${stat.dev.toString(32)}:${stat.ino.toString(32)}`;
if (seenLinks.hasOwnProperty(id)) {
Expand All @@ -1595,7 +1605,18 @@ fs.realpathSync = function realpathSync(p, options) {

// resolve the link, then start over
p = pathModule.resolve(resolvedLink, p.slice(pos));
start();

// Skip over roots
m = splitRootRe.exec(p);
pos = m[0].length;
current = m[0];
base = m[0];

// On windows, check that the root exists. On unix there is no need.
if (isWindows && !knownHard[base]) {
fs.lstatSync(base);
knownHard[base] = true;
}
}

if (cache) cache.set(original, p);
Expand All @@ -1608,10 +1629,10 @@ fs.realpath = function realpath(p, options, callback) {
options = getOptions(options, {});
if (handleError((p = getPathFromURL(p)), callback))
return;
if (typeof p !== 'string')
p += '';
if (!nullCheck(p, callback))
return;

p = p.toString('utf8');
p = pathModule.resolve(p);

const seenLinks = {};
Expand All @@ -1626,26 +1647,21 @@ fs.realpath = function realpath(p, options, callback) {
// the partial path scanned in the previous round, with slash
var previous;

start();
var m = splitRootRe.exec(p);
pos = m[0].length;
current = m[0];
base = m[0];
previous = '';

function start() {
// Skip over roots
var m = splitRootRe.exec(p);
pos = m[0].length;
current = m[0];
base = m[0];
previous = '';

// On windows, check that the root exists. On unix there is no need.
if (isWindows && !knownHard[base]) {
fs.lstat(base, function(err) {
if (err) return callback(err);
knownHard[base] = true;
LOOP();
});
} else {
process.nextTick(LOOP);
}
// On windows, check that the root exists. On unix there is no need.
if (isWindows && !knownHard[base]) {
fs.lstat(base, function(err) {
if (err) return callback(err);
knownHard[base] = true;
LOOP();
});
} else {
process.nextTick(LOOP);
}

// walk down the path, swapping out linked pathparts for their real
Expand All @@ -1657,12 +1673,18 @@ fs.realpath = function realpath(p, options, callback) {
}

// find the next part
nextPartRe.lastIndex = pos;
var result = nextPartRe.exec(p);
var result = nextPart(p, pos);
previous = current;
current += result[0];
base = previous + result[1];
pos = nextPartRe.lastIndex;
if (result === -1) {
var last = p.slice(pos);
current += last;
base = previous + last;
pos = p.length;
} else {
current += p.slice(pos, result + 1);
base = previous + p.slice(pos, result);
pos = result + 1;
}

// continue if not a symlink
if (knownHard[base]) {
Expand Down Expand Up @@ -1711,7 +1733,22 @@ fs.realpath = function realpath(p, options, callback) {
function gotResolvedLink(resolvedLink) {
// resolve the link, then start over
p = pathModule.resolve(resolvedLink, p.slice(pos));
start();
var m = splitRootRe.exec(p);
pos = m[0].length;
current = m[0];
base = m[0];
previous = '';

// On windows, check that the root exists. On unix there is no need.
if (isWindows && !knownHard[base]) {
fs.lstat(base, function(err) {
if (err) return callback(err);
knownHard[base] = true;
LOOP();
});
} else {
process.nextTick(LOOP);
}
}
};
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of interest how much perf gain did we get from this one ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the async realpath()? I didn't measure it explicitly because it's not used for module lookup, I merely made the changes to mirror the realpathSync() changes for consistency.


Expand Down
13 changes: 6 additions & 7 deletions lib/internal/module.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,22 @@ exports = module.exports = {

exports.requireDepth = 0;

// Invoke with makeRequireFunction.call(module) where |module| is the
// Module object to use as the context for the require() function.
function makeRequireFunction() {
const Module = this.constructor;
const self = this;
// Invoke with makeRequireFunction(module) where |module| is the Module object
// to use as the context for the require() function.
function makeRequireFunction(mod) {
const Module = mod.constructor;

function require(path) {
try {
exports.requireDepth += 1;
return self.require(path);
return mod.require(path);
} finally {
exports.requireDepth -= 1;
}
}

function resolve(request) {
return Module._resolveFilename(request, self);
return Module._resolveFilename(request, mod);
}

require.resolve = resolve;
Expand Down
Loading