fs: improve readFile performance
This increases the maximum buffer size per read to 512kb when using `fs.readFile`. This is important to improve the read performance for bigger files. PR-URL: https://github.com/nodejs/node/pull/27063 Refs: https://github.com/nodejs/node/issues/25741 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: Jamie Davis <davisjam@vt.edu>
This commit is contained in:
parent
97c0a34935
commit
eb2d4161f5
10
lib/fs.js
10
lib/fs.js
@ -266,19 +266,17 @@ function readFileAfterStat(err, stats) {
|
|||||||
|
|
||||||
const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0;
|
const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0;
|
||||||
|
|
||||||
if (size === 0) {
|
|
||||||
context.buffers = [];
|
|
||||||
context.read();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (size > kMaxLength) {
|
if (size > kMaxLength) {
|
||||||
err = new ERR_FS_FILE_TOO_LARGE(size);
|
err = new ERR_FS_FILE_TOO_LARGE(size);
|
||||||
return context.close(err);
|
return context.close(err);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (size === 0) {
|
||||||
|
context.buffers = [];
|
||||||
|
} else {
|
||||||
context.buffer = Buffer.allocUnsafeSlow(size);
|
context.buffer = Buffer.allocUnsafeSlow(size);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return context.close(err);
|
return context.close(err);
|
||||||
}
|
}
|
||||||
|
@ -6,7 +6,14 @@ const { Buffer } = require('buffer');
|
|||||||
|
|
||||||
const { FSReqCallback, close, read } = internalBinding('fs');
|
const { FSReqCallback, close, read } = internalBinding('fs');
|
||||||
|
|
||||||
const kReadFileBufferLength = 8 * 1024;
|
// Use 64kb in case the file type is not a regular file and thus do not know the
|
||||||
|
// actual file size. Increasing the value further results in more frequent over
|
||||||
|
// allocation for small files and consumes CPU time and memory that should be
|
||||||
|
// used else wise.
|
||||||
|
// Use up to 512kb per read otherwise to partition reading big files to prevent
|
||||||
|
// blocking other threads in case the available threads are all in use.
|
||||||
|
const kReadFileUnknownBufferLength = 64 * 1024;
|
||||||
|
const kReadFileBufferLength = 512 * 1024;
|
||||||
|
|
||||||
function readFileAfterRead(err, bytesRead) {
|
function readFileAfterRead(err, bytesRead) {
|
||||||
const context = this.context;
|
const context = this.context;
|
||||||
@ -14,19 +21,17 @@ function readFileAfterRead(err, bytesRead) {
|
|||||||
if (err)
|
if (err)
|
||||||
return context.close(err);
|
return context.close(err);
|
||||||
|
|
||||||
if (bytesRead === 0)
|
|
||||||
return context.close();
|
|
||||||
|
|
||||||
context.pos += bytesRead;
|
context.pos += bytesRead;
|
||||||
|
|
||||||
if (context.size !== 0) {
|
if (context.pos === context.size || bytesRead === 0) {
|
||||||
if (context.pos === context.size)
|
|
||||||
context.close();
|
context.close();
|
||||||
else
|
|
||||||
context.read();
|
|
||||||
} else {
|
} else {
|
||||||
|
if (context.size === 0) {
|
||||||
// Unknown size, just read until we don't get bytes.
|
// Unknown size, just read until we don't get bytes.
|
||||||
context.buffers.push(context.buffer.slice(0, bytesRead));
|
const buffer = bytesRead === kReadFileUnknownBufferLength ?
|
||||||
|
context.buffer : context.buffer.slice(0, bytesRead);
|
||||||
|
context.buffers.push(buffer);
|
||||||
|
}
|
||||||
context.read();
|
context.read();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -60,7 +65,7 @@ class ReadFileContext {
|
|||||||
constructor(callback, encoding) {
|
constructor(callback, encoding) {
|
||||||
this.fd = undefined;
|
this.fd = undefined;
|
||||||
this.isUserFd = undefined;
|
this.isUserFd = undefined;
|
||||||
this.size = undefined;
|
this.size = 0;
|
||||||
this.callback = callback;
|
this.callback = callback;
|
||||||
this.buffers = null;
|
this.buffers = null;
|
||||||
this.buffer = null;
|
this.buffer = null;
|
||||||
@ -75,9 +80,10 @@ class ReadFileContext {
|
|||||||
let length;
|
let length;
|
||||||
|
|
||||||
if (this.size === 0) {
|
if (this.size === 0) {
|
||||||
buffer = this.buffer = Buffer.allocUnsafeSlow(kReadFileBufferLength);
|
buffer = Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength);
|
||||||
offset = 0;
|
offset = 0;
|
||||||
length = kReadFileBufferLength;
|
length = kReadFileUnknownBufferLength;
|
||||||
|
this.buffer = buffer;
|
||||||
} else {
|
} else {
|
||||||
buffer = this.buffer;
|
buffer = this.buffer;
|
||||||
offset = this.pos;
|
offset = this.pos;
|
||||||
|
Loading…
x
Reference in New Issue
Block a user