fs: implement mkdir recursive (mkdirp)

Implements mkdirp functionality in node_file.cc. The Benefit
of implementing in C++ layer is that the logic is more easily
shared between the Promise and callback implementation and
there are notable performance improvements.

This commit is part of the Tooling Group Initiative.

Refs: https://github.com/nodejs/user-feedback/pull/70

PR-URL: https://github.com/nodejs/node/pull/21875
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Jon Moss <me@jonathanmoss.me>
Reviewed-By: Ron Korving <ron@ronkorving.nl>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Michael Dawson <michael_dawson@ca.ibm.com>
Reviewed-By: Anatoli Papirovski <apapirovski@mac.com>
Reviewed-By: Sam Ruby <rubys@intertwingly.net>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
This commit is contained in:
Benjamin Coe 2018-08-09 16:52:41 -07:00
parent e0395247c8
commit bdef1b1eb4
No known key found for this signature in database
GPG Key ID: 7668A2653280F496
8 changed files with 442 additions and 37 deletions

View File

@ -0,0 +1,23 @@
'use strict';
const common = require('../common');
const fs = require('fs');
const tmpdir = require('../../test/common/tmpdir');
tmpdir.refresh();
let dirc = 0;
const bench = common.createBenchmark(main, {
n: [1e4],
});
function main({ n }) {
bench.start();
(function r(cntr) {
if (cntr-- <= 0)
return bench.end(n);
const pathname = `${tmpdir.path}/${++dirc}/${++dirc}/${++dirc}/${++dirc}`;
fs.mkdir(pathname, { createParents: true }, (err) => {
r(cntr);
});
}(n));
}

View File

@ -2047,7 +2047,7 @@ changes:
Synchronous lstat(2).
## fs.mkdir(path[, mode], callback)
## fs.mkdir(path[, options], callback)
<!-- YAML
added: v0.1.8
changes:
@ -2066,16 +2066,29 @@ changes:
-->
* `path` {string|Buffer|URL}
* `mode` {integer} Not supported on Windows. **Default:** `0o777`.
* `options` {Object|integer}
* `recursive` {boolean} **Default:** `false`
* `mode` {integer} Not supported on Windows. **Default:** `0o777`.
* `callback` {Function}
* `err` {Error}
Asynchronously creates a directory. No arguments other than a possible exception
are given to the completion callback.
The optional `options` argument can be an integer specifying mode (permission
and sticky bits), or an object with a `mode` property and a `recursive`
property indicating whether parent folders should be created.
```js
// Creates /tmp/a/apple, regardless of whether `/tmp` and /tmp/a exist.
fs.mkdir('/tmp/a/apple', { recursive: true }, (err) => {
if (err) throw err;
});
```
See also: mkdir(2).
## fs.mkdirSync(path[, mode])
## fs.mkdirSync(path[, options])
<!-- YAML
added: v0.1.21
changes:
@ -2086,7 +2099,9 @@ changes:
-->
* `path` {string|Buffer|URL}
* `mode` {integer} Not supported on Windows. **Default:** `0o777`.
* `options` {Object|integer}
* `recursive` {boolean} **Default:** `false`
* `mode` {integer} Not supported on Windows. **Default:** `0o777`.
Synchronously creates a directory. Returns `undefined`.
This is the synchronous version of [`fs.mkdir()`][].
@ -3979,18 +3994,24 @@ changes:
Asynchronous lstat(2). The `Promise` is resolved with the [`fs.Stats`][] object
for the given symbolic link `path`.
### fsPromises.mkdir(path[, mode])
### fsPromises.mkdir(path[, options])
<!-- YAML
added: v10.0.0
-->
* `path` {string|Buffer|URL}
* `mode` {integer} **Default:** `0o777`
* `options` {Object|integer}
* `recursive` {boolean} **Default:** `false`
* `mode` {integer} Not supported on Windows. **Default:** `0o777`.
* Returns: {Promise}
Asynchronously creates a directory then resolves the `Promise` with no
arguments upon success.
The optional `options` argument can be an integer specifying mode (permission
and sticky bits), or an object with a `mode` property and a `recursive`
property indicating whether parent folders should be created.
### fsPromises.mkdtemp(prefix[, options])
<!-- YAML
added: v10.0.0
@ -4627,7 +4648,7 @@ the file contents.
[`fs.ftruncate()`]: #fs_fs_ftruncate_fd_len_callback
[`fs.futimes()`]: #fs_fs_futimes_fd_atime_mtime_callback
[`fs.lstat()`]: #fs_fs_lstat_path_options_callback
[`fs.mkdir()`]: #fs_fs_mkdir_path_mode_callback
[`fs.mkdir()`]: #fs_fs_mkdir_path_options_callback
[`fs.mkdtemp()`]: #fs_fs_mkdtemp_prefix_options_callback
[`fs.open()`]: #fs_fs_open_path_flags_mode_callback
[`fs.read()`]: #fs_fs_read_fd_buffer_offset_length_position_callback

View File

@ -721,29 +721,48 @@ function fsyncSync(fd) {
handleErrorFromBinding(ctx);
}
function mkdir(path, mode, callback) {
path = getPathFromURL(path);
validatePath(path);
if (arguments.length < 3) {
callback = makeCallback(mode);
mode = 0o777;
} else {
callback = makeCallback(callback);
mode = validateMode(mode, 'mode', 0o777);
function mkdir(path, options, callback) {
if (typeof options === 'function') {
callback = options;
options = {};
} else if (typeof options === 'number' || typeof options === 'string') {
options = { mode: options };
}
const {
recursive = false,
mode = 0o777
} = options || {};
callback = makeCallback(callback);
path = getPathFromURL(path);
validatePath(path);
if (typeof recursive !== 'boolean')
throw new ERR_INVALID_ARG_TYPE('recursive', 'boolean', recursive);
const req = new FSReqCallback();
req.oncomplete = callback;
binding.mkdir(pathModule.toNamespacedPath(path), mode, req);
binding.mkdir(pathModule.toNamespacedPath(path),
validateMode(mode, 'mode', 0o777), recursive, req);
}
function mkdirSync(path, mode) {
function mkdirSync(path, options) {
if (typeof options === 'number' || typeof options === 'string') {
options = { mode: options };
}
path = getPathFromURL(path);
const {
recursive = false,
mode = 0o777
} = options || {};
validatePath(path);
mode = validateMode(mode, 'mode', 0o777);
if (typeof recursive !== 'boolean')
throw new ERR_INVALID_ARG_TYPE('recursive', 'boolean', recursive);
const ctx = { path };
binding.mkdir(pathModule.toNamespacedPath(path), mode, undefined, ctx);
binding.mkdir(pathModule.toNamespacedPath(path),
validateMode(mode, 'mode', 0o777), recursive, undefined,
ctx);
handleErrorFromBinding(ctx);
}

View File

@ -289,11 +289,23 @@ async function fsync(handle) {
return binding.fsync(handle.fd, kUsePromises);
}
async function mkdir(path, mode) {
async function mkdir(path, options) {
if (typeof options === 'number' || typeof options === 'string') {
options = { mode: options };
}
const {
recursive = false,
mode = 0o777
} = options || {};
path = getPathFromURL(path);
validatePath(path);
mode = validateMode(mode, 'mode', 0o777);
return binding.mkdir(pathModule.toNamespacedPath(path), mode, kUsePromises);
if (typeof recursive !== 'boolean')
throw new ERR_INVALID_ARG_TYPE('recursive', 'boolean', recursive);
return binding.mkdir(pathModule.toNamespacedPath(path),
validateMode(mode, 'mode', 0o777), recursive,
kUsePromises);
}
async function readdir(path, options) {

View File

@ -76,6 +76,16 @@ using v8::Value;
# define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif
#ifndef S_ISDIR
# define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR)
#endif
#ifdef __POSIX__
const char* kPathSeparator = "/";
#else
const char* kPathSeparator = "\\/";
#endif
#define GET_OFFSET(a) ((a)->IsNumber() ? (a).As<Integer>()->Value() : -1)
#define TRACE_NAME(name) "fs.sync." #name
#define GET_TRACE_ENABLED \
@ -1148,11 +1158,137 @@ static void RMDir(const FunctionCallbackInfo<Value>& args) {
}
}
int MKDirpSync(uv_loop_t* loop, uv_fs_t* req, const std::string& path, int mode,
uv_fs_cb cb = nullptr) {
FSContinuationData continuation_data(req, mode, cb);
continuation_data.PushPath(std::move(path));
while (continuation_data.paths.size() > 0) {
std::string next_path = continuation_data.PopPath();
int err = uv_fs_mkdir(loop, req, next_path.c_str(), mode, nullptr);
while (true) {
switch (err) {
case 0:
if (continuation_data.paths.size() == 0) {
return 0;
}
break;
case UV_ENOENT: {
std::string dirname = next_path.substr(0,
next_path.find_last_of(kPathSeparator));
if (dirname != next_path) {
continuation_data.PushPath(std::move(next_path));
continuation_data.PushPath(std::move(dirname));
} else if (continuation_data.paths.size() == 0) {
err = UV_EEXIST;
continue;
}
break;
}
case UV_EPERM: {
return err;
}
default:
uv_fs_req_cleanup(req);
err = uv_fs_stat(loop, req, next_path.c_str(), nullptr);
if (err == 0 && !S_ISDIR(req->statbuf.st_mode)) return UV_EEXIST;
if (err < 0) return err;
break;
}
break;
}
uv_fs_req_cleanup(req);
}
return 0;
}
int MKDirpAsync(uv_loop_t* loop,
uv_fs_t* req,
const char* path,
int mode,
uv_fs_cb cb) {
FSReqBase* req_wrap = FSReqBase::from_req(req);
// on the first iteration of algorithm, stash state information.
if (req_wrap->continuation_data == nullptr) {
req_wrap->continuation_data = std::unique_ptr<FSContinuationData>{
new FSContinuationData(req, mode, cb)};
req_wrap->continuation_data->PushPath(std::move(path));
}
// on each iteration of algorithm, mkdir directory on top of stack.
std::string next_path = req_wrap->continuation_data->PopPath();
int err = uv_fs_mkdir(loop, req, next_path.c_str(), mode,
uv_fs_callback_t{[](uv_fs_t* req) {
FSReqBase* req_wrap = FSReqBase::from_req(req);
Environment* env = req_wrap->env();
uv_loop_t* loop = env->event_loop();
std::string path = req->path;
int err = req->result;
while (true) {
switch (err) {
case 0: {
if (req_wrap->continuation_data->paths.size() == 0) {
req_wrap->continuation_data->Done(0);
} else {
uv_fs_req_cleanup(req);
MKDirpAsync(loop, req, path.c_str(),
req_wrap->continuation_data->mode, nullptr);
}
break;
}
case UV_ENOENT: {
std::string dirname = path.substr(0,
path.find_last_of(kPathSeparator));
if (dirname != path) {
req_wrap->continuation_data->PushPath(std::move(path));
req_wrap->continuation_data->PushPath(std::move(dirname));
} else if (req_wrap->continuation_data->paths.size() == 0) {
err = UV_EEXIST;
continue;
}
uv_fs_req_cleanup(req);
MKDirpAsync(loop, req, path.c_str(),
req_wrap->continuation_data->mode, nullptr);
break;
}
case UV_EPERM: {
req_wrap->continuation_data->Done(err);
break;
}
default:
if (err == UV_EEXIST &&
req_wrap->continuation_data->paths.size() > 0) {
uv_fs_req_cleanup(req);
MKDirpAsync(loop, req, path.c_str(),
req_wrap->continuation_data->mode, nullptr);
} else {
// verify that the path pointed to is actually a directory.
uv_fs_req_cleanup(req);
int err = uv_fs_stat(loop, req, path.c_str(),
uv_fs_callback_t{[](uv_fs_t* req) {
FSReqBase* req_wrap = FSReqBase::from_req(req);
int err = req->result;
if (err == 0 && !S_ISDIR(req->statbuf.st_mode)) err = UV_EEXIST;
req_wrap->continuation_data->Done(err);
}});
if (err < 0) req_wrap->continuation_data->Done(err);
}
break;
}
break;
}
}});
return err;
}
static void MKDir(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
const int argc = args.Length();
CHECK_GE(argc, 3);
CHECK_GE(argc, 4);
BufferValue path(env->isolate(), args[0]);
CHECK_NOT_NULL(*path);
@ -1160,16 +1296,24 @@ static void MKDir(const FunctionCallbackInfo<Value>& args) {
CHECK(args[1]->IsInt32());
const int mode = args[1].As<Int32>()->Value();
FSReqBase* req_wrap_async = GetReqWrap(env, args[2]);
CHECK(args[2]->IsBoolean());
bool mkdirp = args[2]->IsTrue();
FSReqBase* req_wrap_async = GetReqWrap(env, args[3]);
if (req_wrap_async != nullptr) { // mkdir(path, mode, req)
AsyncCall(env, req_wrap_async, args, "mkdir", UTF8, AfterNoArgs,
uv_fs_mkdir, *path, mode);
AsyncCall(env, req_wrap_async, args, "mkdir", UTF8,
AfterNoArgs, mkdirp ? MKDirpAsync : uv_fs_mkdir, *path, mode);
} else { // mkdir(path, mode, undefined, ctx)
CHECK_EQ(argc, 4);
CHECK_EQ(argc, 5);
FSReqWrapSync req_wrap_sync;
FS_SYNC_TRACE_BEGIN(mkdir);
SyncCall(env, args[3], &req_wrap_sync, "mkdir",
uv_fs_mkdir, *path, mode);
if (mkdirp) {
SyncCall(env, args[4], &req_wrap_sync, "mkdir",
MKDirpSync, *path, mode);
} else {
SyncCall(env, args[4], &req_wrap_sync, "mkdir",
uv_fs_mkdir, *path, mode);
}
FS_SYNC_TRACE_END(mkdir);
}
}

View File

@ -21,10 +21,50 @@ using v8::Value;
namespace fs {
// structure used to store state during a complex operation, e.g., mkdirp.
class FSContinuationData : public MemoryRetainer {
public:
FSContinuationData(uv_fs_t* req, int mode, uv_fs_cb done_cb)
: req(req), mode(mode), done_cb(done_cb) {
}
uv_fs_t* req;
int mode;
std::vector<std::string> paths;
void PushPath(std::string&& path) {
paths.emplace_back(std::move(path));
}
void PushPath(const std::string& path) {
paths.push_back(path);
}
std::string PopPath() {
CHECK_GT(paths.size(), 0);
std::string path = std::move(paths.back());
paths.pop_back();
return path;
}
void Done(int result) {
req->result = result;
done_cb(req);
}
void MemoryInfo(MemoryTracker* tracker) const override {
tracker->TrackThis(this);
tracker->TrackField("paths", paths);
}
private:
uv_fs_cb done_cb;
};
class FSReqBase : public ReqWrap<uv_fs_t> {
public:
typedef MaybeStackBuffer<char, 64> FSReqBuffer;
std::unique_ptr<FSContinuationData> continuation_data = nullptr;
FSReqBase(Environment* env, Local<Object> req, AsyncWrap::ProviderType type,
bool use_bigint)
@ -97,6 +137,7 @@ class FSReqCallback : public FSReqBase {
void MemoryInfo(MemoryTracker* tracker) const override {
tracker->TrackThis(this);
tracker->TrackField("continuation_data", continuation_data);
}
ADD_MEMORY_INFO_NAME(FSReqCallback)
@ -162,6 +203,7 @@ class FSReqPromise : public FSReqBase {
void MemoryInfo(MemoryTracker* tracker) const override {
tracker->TrackThis(this);
tracker->TrackField("stats_field_array", stats_field_array_);
tracker->TrackField("continuation_data", continuation_data);
}
ADD_MEMORY_INFO_NAME(FSReqPromise)

View File

@ -23,12 +23,18 @@
const common = require('../common');
const assert = require('assert');
const fs = require('fs');
const path = require('path');
const tmpdir = require('../common/tmpdir');
tmpdir.refresh();
let dirc = 0;
function nextdir() {
return `test${++dirc}`;
}
{
const pathname = `${tmpdir.path}/test1`;
const pathname = path.join(tmpdir.path, nextdir());
fs.mkdir(pathname, common.mustCall(function(err) {
assert.strictEqual(err, null);
@ -37,7 +43,7 @@ tmpdir.refresh();
}
{
const pathname = `${tmpdir.path}/test2`;
const pathname = path.join(tmpdir.path, nextdir());
fs.mkdir(pathname, 0o777, common.mustCall(function(err) {
assert.strictEqual(err, null);
@ -46,7 +52,7 @@ tmpdir.refresh();
}
{
const pathname = `${tmpdir.path}/test3`;
const pathname = path.join(tmpdir.path, nextdir());
fs.mkdirSync(pathname);
@ -71,6 +77,101 @@ tmpdir.refresh();
);
});
// mkdirpSync when both top-level, and sub-folders do not exist.
{
const pathname = path.join(tmpdir.path, nextdir(), nextdir());
fs.mkdirSync(pathname, { recursive: true });
const exists = fs.existsSync(pathname);
assert.strictEqual(exists, true);
assert.strictEqual(fs.statSync(pathname).isDirectory(), true);
}
// mkdirpSync when folder already exists.
{
const pathname = path.join(tmpdir.path, nextdir(), nextdir());
fs.mkdirSync(pathname, { recursive: true });
// should not cause an error.
fs.mkdirSync(pathname, { recursive: true });
const exists = fs.existsSync(pathname);
assert.strictEqual(exists, true);
assert.strictEqual(fs.statSync(pathname).isDirectory(), true);
}
// mkdirpSync ../
{
const pathname = `${tmpdir.path}/${nextdir()}/../${nextdir()}/${nextdir()}`;
fs.mkdirSync(pathname, { recursive: true });
const exists = fs.existsSync(pathname);
assert.strictEqual(exists, true);
assert.strictEqual(fs.statSync(pathname).isDirectory(), true);
}
// mkdirpSync when path is a file.
{
const pathname = path.join(tmpdir.path, nextdir(), nextdir());
fs.mkdirSync(path.dirname(pathname));
fs.writeFileSync(pathname, '', 'utf8');
try {
fs.mkdirSync(pathname, { recursive: true });
throw new Error('unreachable');
} catch (err) {
assert.notStrictEqual(err.message, 'unreachable');
assert.strictEqual(err.code, 'EEXIST');
assert.strictEqual(err.syscall, 'mkdir');
}
}
// mkdirp when folder does not yet exist.
{
const pathname = path.join(tmpdir.path, nextdir(), nextdir());
fs.mkdir(pathname, { recursive: true }, common.mustCall(function(err) {
assert.strictEqual(err, null);
assert.strictEqual(fs.existsSync(pathname), true);
assert.strictEqual(fs.statSync(pathname).isDirectory(), true);
}));
}
// mkdirp when path is a file.
{
const pathname = path.join(tmpdir.path, nextdir(), nextdir());
fs.mkdirSync(path.dirname(pathname));
fs.writeFileSync(pathname, '', 'utf8');
fs.mkdir(pathname, { recursive: true }, (err) => {
assert.strictEqual(err.code, 'EEXIST');
assert.strictEqual(err.syscall, 'mkdir');
assert.strictEqual(fs.statSync(pathname).isDirectory(), false);
});
}
// mkdirpSync dirname loop
// XXX: windows and smartos have issues removing a directory that you're in.
if (common.isMainThread && (common.isLinux || common.isOSX)) {
const pathname = path.join(tmpdir.path, nextdir());
fs.mkdirSync(pathname);
process.chdir(pathname);
fs.rmdirSync(pathname);
try {
fs.mkdirSync('X', { recursive: true });
throw new Error('unreachable');
} catch (err) {
assert.notStrictEqual(err.message, 'unreachable');
assert.strictEqual(err.code, 'ENOENT');
assert.strictEqual(err.syscall, 'mkdir');
}
fs.mkdir('X', { recursive: true }, (err) => {
assert.strictEqual(err.code, 'ENOENT');
assert.strictEqual(err.syscall, 'mkdir');
});
}
// Keep the event loop alive so the async mkdir() requests
// have a chance to run (since they don't ref the event loop).
process.nextTick(() => {});

View File

@ -29,11 +29,17 @@ const {
symlink,
truncate,
unlink,
utimes
utimes,
writeFile
} = fsPromises;
const tmpDir = tmpdir.path;
let dirc = 0;
function nextdir() {
return `test${++dirc}`;
}
// fs.promises should not be enumerable as long as it causes a warning to be
// emitted.
assert.strictEqual(Object.keys(fs).includes('promises'), false);
@ -201,12 +207,49 @@ function verifyStatObject(stat) {
await mkdir(newdir);
stats = await stat(newdir);
assert(stats.isDirectory());
const list = await readdir(tmpDir);
assert.deepStrictEqual(list, ['baz2.js', 'dir']);
await rmdir(newdir);
// mkdirp when folder does not yet exist.
{
const dir = path.join(tmpDir, nextdir(), nextdir());
await mkdir(dir, { recursive: true });
stats = await stat(dir);
assert(stats.isDirectory());
}
// mkdirp when path is a file.
{
const dir = path.join(tmpDir, nextdir(), nextdir());
await mkdir(path.dirname(dir));
await writeFile(dir);
try {
await mkdir(dir, { recursive: true });
throw new Error('unreachable');
} catch (err) {
assert.notStrictEqual(err.message, 'unreachable');
assert.strictEqual(err.code, 'EEXIST');
assert.strictEqual(err.syscall, 'mkdir');
}
}
// mkdirp ./
{
const dir = path.resolve(tmpDir, `${nextdir()}/./${nextdir()}`);
await mkdir(dir, { recursive: true });
stats = await stat(dir);
assert(stats.isDirectory());
}
// mkdirp ../
{
const dir = path.resolve(tmpDir, `${nextdir()}/../${nextdir()}`);
await mkdir(dir, { recursive: true });
stats = await stat(dir);
assert(stats.isDirectory());
}
await mkdtemp(path.resolve(tmpDir, 'FOO'));
assert.rejects(
// mkdtemp() expects to get a string prefix.