fs: improve readFile performance

This increases the maximum buffer size per read to 512kb when using
`fs.readFile`. This is important to improve the read performance for
bigger files.

PR-URL: https://github.com/nodejs/node/pull/27063
Refs: https://github.com/nodejs/node/issues/25741
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Jamie Davis <davisjam@vt.edu>
This commit is contained in:
Ruben Bridgewater 2019-04-03 02:58:31 +02:00
parent 97c0a34935
commit eb2d4161f5
No known key found for this signature in database
GPG Key ID: F07496B3EB3C1762
2 changed files with 25 additions and 21 deletions

View File

@ -266,19 +266,17 @@ function readFileAfterStat(err, stats) {
const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0; const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0;
if (size === 0) {
context.buffers = [];
context.read();
return;
}
if (size > kMaxLength) { if (size > kMaxLength) {
err = new ERR_FS_FILE_TOO_LARGE(size); err = new ERR_FS_FILE_TOO_LARGE(size);
return context.close(err); return context.close(err);
} }
try { try {
if (size === 0) {
context.buffers = [];
} else {
context.buffer = Buffer.allocUnsafeSlow(size); context.buffer = Buffer.allocUnsafeSlow(size);
}
} catch (err) { } catch (err) {
return context.close(err); return context.close(err);
} }

View File

@ -6,7 +6,14 @@ const { Buffer } = require('buffer');
const { FSReqCallback, close, read } = internalBinding('fs'); const { FSReqCallback, close, read } = internalBinding('fs');
const kReadFileBufferLength = 8 * 1024; // Use 64kb in case the file type is not a regular file and thus do not know the
// actual file size. Increasing the value further results in more frequent over
// allocation for small files and consumes CPU time and memory that should be
// used else wise.
// Use up to 512kb per read otherwise to partition reading big files to prevent
// blocking other threads in case the available threads are all in use.
const kReadFileUnknownBufferLength = 64 * 1024;
const kReadFileBufferLength = 512 * 1024;
function readFileAfterRead(err, bytesRead) { function readFileAfterRead(err, bytesRead) {
const context = this.context; const context = this.context;
@ -14,19 +21,17 @@ function readFileAfterRead(err, bytesRead) {
if (err) if (err)
return context.close(err); return context.close(err);
if (bytesRead === 0)
return context.close();
context.pos += bytesRead; context.pos += bytesRead;
if (context.size !== 0) { if (context.pos === context.size || bytesRead === 0) {
if (context.pos === context.size)
context.close(); context.close();
else
context.read();
} else { } else {
if (context.size === 0) {
// Unknown size, just read until we don't get bytes. // Unknown size, just read until we don't get bytes.
context.buffers.push(context.buffer.slice(0, bytesRead)); const buffer = bytesRead === kReadFileUnknownBufferLength ?
context.buffer : context.buffer.slice(0, bytesRead);
context.buffers.push(buffer);
}
context.read(); context.read();
} }
} }
@ -60,7 +65,7 @@ class ReadFileContext {
constructor(callback, encoding) { constructor(callback, encoding) {
this.fd = undefined; this.fd = undefined;
this.isUserFd = undefined; this.isUserFd = undefined;
this.size = undefined; this.size = 0;
this.callback = callback; this.callback = callback;
this.buffers = null; this.buffers = null;
this.buffer = null; this.buffer = null;
@ -75,9 +80,10 @@ class ReadFileContext {
let length; let length;
if (this.size === 0) { if (this.size === 0) {
buffer = this.buffer = Buffer.allocUnsafeSlow(kReadFileBufferLength); buffer = Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength);
offset = 0; offset = 0;
length = kReadFileBufferLength; length = kReadFileUnknownBufferLength;
this.buffer = buffer;
} else { } else {
buffer = this.buffer; buffer = this.buffer;
offset = this.pos; offset = this.pos;