src: move package resolver to c++

Co-authored-by: Daniel Lemire <daniel@lemire.me>
PR-URL: https://github.com/nodejs/node/pull/50322
Reviewed-By: Jacob Smith <jacob@frende.me>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Geoffrey Booth <webadmin@geoffreybooth.com>
This commit is contained in:
Yagiz Nizipli 2023-10-25 17:51:27 -04:00 committed by Node.js GitHub Bot
parent 4ec085b240
commit f13dbfd43a
26 changed files with 804 additions and 353 deletions

View File

@ -427,7 +427,7 @@ ObjectDefineProperty(Module, '_readPackage', {
* @param {string} originalPath The specifier passed to `require`
*/
function tryPackage(requestPath, exts, isMain, originalPath) {
const pkg = _readPackage(requestPath).main;
const { main: pkg, pjsonPath } = _readPackage(requestPath);
if (!pkg) {
return tryExtensions(path.resolve(requestPath, 'index'), exts, isMain);
@ -446,14 +446,13 @@ function tryPackage(requestPath, exts, isMain, originalPath) {
'Please verify that the package.json has a valid "main" entry',
);
err.code = 'MODULE_NOT_FOUND';
err.path = path.resolve(requestPath, 'package.json');
err.path = pjsonPath;
err.requestPath = originalPath;
// TODO(BridgeAR): Add the requireStack as well.
throw err;
} else {
const jsonPath = path.resolve(requestPath, 'package.json');
process.emitWarning(
`Invalid 'main' field in '${jsonPath}' of '${pkg}'. ` +
`Invalid 'main' field in '${pjsonPath}' of '${pkg}'. ` +
'Please either fix that or report it to the module author',
'DeprecationWarning',
'DEP0128',
@ -539,16 +538,16 @@ function trySelfParentPath(parent) {
function trySelf(parentPath, request) {
if (!parentPath) { return false; }
const { data: pkg, path: pkgPath } = packageJsonReader.readPackageScope(parentPath);
if (!pkg || pkg.exports == null || pkg.name === undefined) {
const pkg = packageJsonReader.getNearestParentPackageJSON(parentPath);
if (pkg?.data.exports === undefined || pkg.data.name === undefined) {
return false;
}
let expansion;
if (request === pkg.name) {
if (request === pkg.data.name) {
expansion = '.';
} else if (StringPrototypeStartsWith(request, `${pkg.name}/`)) {
expansion = '.' + StringPrototypeSlice(request, pkg.name.length);
} else if (StringPrototypeStartsWith(request, `${pkg.data.name}/`)) {
expansion = '.' + StringPrototypeSlice(request, pkg.data.name.length);
} else {
return false;
}
@ -556,11 +555,11 @@ function trySelf(parentPath, request) {
try {
const { packageExportsResolve } = require('internal/modules/esm/resolve');
return finalizeEsmResolution(packageExportsResolve(
pathToFileURL(pkgPath + '/package.json'), expansion, pkg,
pathToFileURL(parentPath), getCjsConditions()), parentPath, pkgPath);
pathToFileURL(pkg.path + '/package.json'), expansion, pkg.data,
pathToFileURL(parentPath), getCjsConditions()), parentPath, pkg.path);
} catch (e) {
if (e.code === 'ERR_MODULE_NOT_FOUND') {
throw createEsmNotFoundErr(request, pkgPath + '/package.json');
throw createEsmNotFoundErr(request, pkg.path + '/package.json');
}
throw e;
}
@ -1099,7 +1098,7 @@ Module._resolveFilename = function(request, parent, isMain, options) {
if (request[0] === '#' && (parent?.filename || parent?.id === '<repl>')) {
const parentPath = parent?.filename ?? process.cwd() + path.sep;
const pkg = packageJsonReader.readPackageScope(parentPath) || { __proto__: null };
const pkg = packageJsonReader.getNearestParentPackageJSON(parentPath) || { __proto__: null };
if (pkg.data?.imports != null) {
try {
const { packageImportsResolve } = require('internal/modules/esm/resolve');
@ -1397,9 +1396,9 @@ Module._extensions['.js'] = function(module, filename) {
content = fs.readFileSync(filename, 'utf8');
}
if (StringPrototypeEndsWith(filename, '.js')) {
const pkg = packageJsonReader.readPackageScope(filename) || { __proto__: null };
const pkg = packageJsonReader.getNearestParentPackageJSON(filename);
// Function require shouldn't be used in ES modules.
if (pkg.data?.type === 'module') {
if (pkg?.data.type === 'module') {
// This is an error path because `require` of a `.js` file in a `"type": "module"` scope is not allowed.
const parent = moduleParentCache.get(module);
const parentPath = parent?.filename;

View File

@ -19,7 +19,7 @@ const {
const experimentalNetworkImports =
getOptionValue('--experimental-network-imports');
const { containsModuleSyntax } = internalBinding('contextify');
const { getPackageType } = require('internal/modules/esm/resolve');
const { getPackageType } = require('internal/modules/esm/package_config');
const { fileURLToPath } = require('internal/url');
const { ERR_UNKNOWN_FILE_EXTENSION } = require('internal/errors').codes;

View File

@ -228,7 +228,7 @@ class ModuleJob {
const packageConfig =
StringPrototypeStartsWith(this.module.url, 'file://') &&
RegExpPrototypeExec(/\.js(\?[^#]*)?(#.*)?$/, this.module.url) !== null &&
require('internal/modules/esm/resolve')
require('internal/modules/esm/package_config')
.getPackageScopeConfig(this.module.url);
if (packageConfig.type === 'module') {
e.message +=

View File

@ -1,69 +1,44 @@
'use strict';
const {
StringPrototypeEndsWith,
} = primordials;
const { URL, fileURLToPath } = require('internal/url');
const packageJsonReader = require('internal/modules/package_json_reader');
const { ArrayIsArray } = primordials;
const modulesBinding = internalBinding('modules');
const { deserializePackageJSON } = require('internal/modules/package_json_reader');
/**
* @typedef {object} PackageConfig
* @property {string} pjsonPath - The path to the package.json file.
* @property {boolean} exists - Whether the package.json file exists.
* @property {'none' | 'commonjs' | 'module'} type - The type of the package.
* @property {string} [name] - The name of the package.
* @property {string} [main] - The main entry point of the package.
* @property {PackageTarget} [exports] - The exports configuration of the package.
* @property {Record<string, string | Record<string, string>>} [imports] - The imports configuration of the package.
*/
/**
* @typedef {string | string[] | Record<string, string | Record<string, string>>} PackageTarget
*/
// TODO(@anonrig): Merge this file with internal/esm/package_json_reader.js
/**
* Returns the package configuration for the given resolved URL.
* @param {URL | string} resolved - The resolved URL.
* @returns {PackageConfig} - The package configuration.
* @returns {import('typings/internalBinding/modules').PackageConfig} - The package configuration.
*/
function getPackageScopeConfig(resolved) {
let packageJSONUrl = new URL('./package.json', resolved);
while (true) {
const packageJSONPath = packageJSONUrl.pathname;
if (StringPrototypeEndsWith(packageJSONPath, 'node_modules/package.json')) {
break;
}
const packageConfig = packageJsonReader.read(fileURLToPath(packageJSONUrl), {
__proto__: null,
specifier: resolved,
isESM: true,
});
if (packageConfig.exists) {
return packageConfig;
const result = modulesBinding.getPackageScopeConfig(`${resolved}`);
if (ArrayIsArray(result)) {
return deserializePackageJSON(`${resolved}`, result, false /* checkIntegrity */);
}
const lastPackageJSONUrl = packageJSONUrl;
packageJSONUrl = new URL('../package.json', packageJSONUrl);
// Terminates at root where ../package.json equals ../../package.json
// (can't just check "/package.json" for Windows support).
if (packageJSONUrl.pathname === lastPackageJSONUrl.pathname) {
break;
}
}
const packageJSONPath = fileURLToPath(packageJSONUrl);
// This means that the response is a string
// and it is the path to the package.json file
return {
__proto__: null,
pjsonPath: packageJSONPath,
pjsonPath: result,
exists: false,
main: undefined,
name: undefined,
type: 'none',
exports: undefined,
imports: undefined,
};
}
/**
* Returns the package type for a given URL.
* @param {URL} url - The URL to get the package type for.
*/
function getPackageType(url) {
// TODO(@anonrig): Write a C++ function that returns only "type".
return getPackageScopeConfig(url).type;
}
module.exports = {
getPackageScopeConfig,
getPackageType,
};

View File

@ -198,7 +198,7 @@ const legacyMainResolveExtensionsIndexes = {
* 4. TRY(pkg_url/index.js, pkg_url/index.json, pkg_url/index.node)
* 5. NOT_FOUND
* @param {URL} packageJSONUrl
* @param {PackageConfig} packageConfig
* @param {import('typings/internalBinding/modules').PackageConfig} packageConfig
* @param {string | URL | undefined} base
* @returns {URL}
*/
@ -502,7 +502,7 @@ function resolvePackageTarget(packageJSONUrl, target, subpath, packageSubpath,
}
return resolveResult;
}
if (lastException === undefined || lastException === null) {
if (lastException == null) {
return lastException;
}
throw lastException;
@ -575,7 +575,7 @@ function isConditionalExportsMainSugar(exports, packageJSONUrl, base) {
*/
function packageExportsResolve(
packageJSONUrl, packageSubpath, packageConfig, base, conditions) {
let exports = packageConfig.exports;
let { exports } = packageConfig;
if (isConditionalExportsMainSugar(exports, packageJSONUrl, base)) {
exports = { '.': exports };
}
@ -740,15 +740,6 @@ function packageImportsResolve(name, base, conditions) {
throw importNotDefined(name, packageJSONUrl, base);
}
/**
* Returns the package type for a given URL.
* @param {URL} url - The URL to get the package type for.
*/
function getPackageType(url) {
const packageConfig = getPackageScopeConfig(url);
return packageConfig.type;
}
/**
* Parse a package name from a specifier.
* @param {string} specifier - The import specifier.
@ -796,6 +787,7 @@ function parsePackageName(specifier, base) {
* @returns {URL} - The resolved URL.
*/
function packageResolve(specifier, base, conditions) {
// TODO(@anonrig): Move this to a C++ function.
if (BuiltinModule.canBeRequiredWithoutScheme(specifier)) {
return new URL('node:' + specifier);
}
@ -1179,8 +1171,6 @@ module.exports = {
decorateErrorWithCommonJSHints,
defaultResolve,
encodedSepRegEx,
getPackageScopeConfig,
getPackageType,
packageExportsResolve,
packageImportsResolve,
throwIfInvalidParentURL,

View File

@ -2,105 +2,22 @@
const {
JSONParse,
ObjectPrototypeHasOwnProperty,
SafeMap,
StringPrototypeEndsWith,
StringPrototypeIndexOf,
StringPrototypeLastIndexOf,
StringPrototypeSlice,
StringPrototypeLastIndexOf,
ObjectDefineProperty,
} = primordials;
const {
ERR_INVALID_PACKAGE_CONFIG,
} = require('internal/errors').codes;
const { internalModuleReadJSON } = internalBinding('fs');
const { resolve, sep, toNamespacedPath } = require('path');
const permission = require('internal/process/permission');
const { kEmptyObject, setOwnProperty } = require('internal/util');
const { fileURLToPath, pathToFileURL } = require('internal/url');
const cache = new SafeMap();
const modulesBinding = internalBinding('modules');
const { resolve, sep } = require('path');
const { kEmptyObject } = require('internal/util');
const { pathToFileURL } = require('internal/url');
let manifest;
/**
* @typedef {{
* exists: boolean,
* pjsonPath: string,
* exports?: string | string[] | Record<string, unknown>,
* imports?: string | string[] | Record<string, unknown>,
* name?: string,
* main?: string,
* type: 'commonjs' | 'module' | 'none',
* }} PackageConfig
*/
/**
* @param {string} jsonPath
* @param {{
* base?: string,
* specifier: string,
* isESM: boolean,
* }} options
* @returns {PackageConfig}
* @param {string} value The integrity value to check against.
*/
function read(jsonPath, { base, specifier, isESM } = kEmptyObject) {
if (cache.has(jsonPath)) {
return cache.get(jsonPath);
}
const string = internalModuleReadJSON(
toNamespacedPath(jsonPath),
);
const result = {
__proto__: null,
exists: false,
pjsonPath: jsonPath,
main: undefined,
name: undefined,
type: 'none', // Ignore unknown types for forwards compatibility
exports: undefined,
imports: undefined,
};
if (string !== undefined) {
let parsed;
try {
parsed = JSONParse(string);
} catch (cause) {
const error = new ERR_INVALID_PACKAGE_CONFIG(
jsonPath,
isESM && (base ? `"${specifier}" from ` : '') + fileURLToPath(base || specifier),
cause.message,
);
setOwnProperty(error, 'cause', cause);
throw error;
}
result.exists = true;
// ObjectPrototypeHasOwnProperty is used to avoid prototype pollution.
if (ObjectPrototypeHasOwnProperty(parsed, 'name') && typeof parsed.name === 'string') {
result.name = parsed.name;
}
if (ObjectPrototypeHasOwnProperty(parsed, 'main') && typeof parsed.main === 'string') {
result.main = parsed.main;
}
if (ObjectPrototypeHasOwnProperty(parsed, 'exports')) {
result.exports = parsed.exports;
}
if (ObjectPrototypeHasOwnProperty(parsed, 'imports')) {
result.imports = parsed.imports;
}
// Ignore unknown types for forwards compatibility
if (ObjectPrototypeHasOwnProperty(parsed, 'type') && (parsed.type === 'commonjs' || parsed.type === 'module')) {
result.type = parsed.type;
}
function checkPackageJSONIntegrity(jsonPath, value) {
if (manifest === undefined) {
const { getOptionValue } = require('internal/options');
manifest = getOptionValue('--experimental-policy') ?
@ -109,54 +26,134 @@ function read(jsonPath, { base, specifier, isESM } = kEmptyObject) {
}
if (manifest !== null) {
const jsonURL = pathToFileURL(jsonPath);
manifest.assertIntegrity(jsonURL, string);
manifest.assertIntegrity(jsonURL, value);
}
}
cache.set(jsonPath, result);
return result;
}
/**
* @param {string} path
* @param {import('typings/internalBinding/modules').SerializedPackageConfig} contents
* @param {boolean} [checkIntegrity=false] Whether to check the integrity of the package.json file.
* @returns {import('typings/internalBinding/modules').PackageConfig}
*/
function deserializePackageJSON(path, contents, checkIntegrity = false) {
if (contents === undefined) {
return {
__proto__: null,
exists: false,
pjsonPath: path,
type: 'none', // Ignore unknown types for forwards compatibility
};
}
let pjsonPath = path;
const {
0: name,
1: main,
2: type,
3: plainImports,
4: plainExports,
5: manifest,
6: optionalFilePath,
} = contents;
// This is required to be used in getPackageScopeConfig.
if (optionalFilePath) {
pjsonPath = optionalFilePath;
}
if (checkIntegrity) {
// parsed[5] is only available when experimental policy is enabled.
checkPackageJSONIntegrity(pjsonPath, manifest);
}
// The imports and exports fields can be either undefined or a string.
// - If it's a string, it's either plain string or a stringified JSON string.
// - If it's a stringified JSON string, it starts with either '[' or '{'.
const requiresJSONParse = (value) => (value !== undefined && (value[0] === '[' || value[0] === '{'));
return {
__proto__: null,
exists: true,
pjsonPath,
name,
main,
type,
// This getters are used to lazily parse the imports and exports fields.
get imports() {
const value = requiresJSONParse(plainImports) ? JSONParse(plainImports) : plainImports;
ObjectDefineProperty(this, 'imports', { __proto__: null, value });
return this.imports;
},
get exports() {
const value = requiresJSONParse(plainExports) ? JSONParse(plainExports) : plainExports;
ObjectDefineProperty(this, 'exports', { __proto__: null, value });
return this.exports;
},
};
}
/**
* Reads a package.json file and returns the parsed contents.
* @param {string} jsonPath
* @param {{
* base?: URL | string,
* specifier?: URL | string,
* isESM?: boolean,
* }} options
* @returns {import('typings/internalBinding/modules').PackageConfig}
*/
function read(jsonPath, { base, specifier, isESM } = kEmptyObject) {
// This function will be called by both CJS and ESM, so we need to make sure
// non-null attributes are converted to strings.
const parsed = modulesBinding.readPackageJSON(
jsonPath,
isESM,
base == null ? undefined : `${base}`,
specifier == null ? undefined : `${specifier}`,
);
return deserializePackageJSON(jsonPath, parsed, true /* checkIntegrity */);
}
/**
* @deprecated Expected to be removed in favor of `read` in the future.
* Behaves the same was as `read`, but appends package.json to the path.
* @param {string} requestPath
* @return {PackageConfig}
*/
function readPackage(requestPath) {
// TODO(@anonrig): Remove this function.
return read(resolve(requestPath, 'package.json'));
}
/**
* Get the nearest parent package.json file from a given path.
* Return the package.json data and the path to the package.json file, or false.
* Return the package.json data and the path to the package.json file, or undefined.
* @param {string} checkPath The path to start searching from.
* @returns {undefined | {data: import('typings/internalBinding/modules').PackageConfig, path: string}}
*/
function readPackageScope(checkPath) {
const rootSeparatorIndex = StringPrototypeIndexOf(checkPath, sep);
let separatorIndex;
const enabledPermission = permission.isEnabled();
do {
separatorIndex = StringPrototypeLastIndexOf(checkPath, sep);
checkPath = StringPrototypeSlice(checkPath, 0, separatorIndex);
// Stop the search when the process doesn't have permissions
// to walk upwards
if (enabledPermission && !permission.has('fs.read', checkPath + sep)) {
return false;
function getNearestParentPackageJSON(checkPath) {
const result = modulesBinding.getNearestParentPackageJSON(checkPath);
if (result === undefined) {
return undefined;
}
if (StringPrototypeEndsWith(checkPath, sep + 'node_modules')) {
return false;
}
const pjson = readPackage(checkPath + sep);
if (pjson.exists) {
return {
data: pjson,
path: checkPath,
};
}
} while (separatorIndex > rootSeparatorIndex);
return false;
const data = deserializePackageJSON(checkPath, result, true /* checkIntegrity */);
// Path should be the root folder of the matched package.json
// For example for ~/path/package.json, it should be ~/path
const path = StringPrototypeSlice(data.pjsonPath, 0, StringPrototypeLastIndexOf(data.pjsonPath, sep));
return { data, path };
}
module.exports = {
checkPackageJSONIntegrity,
read,
readPackage,
readPackageScope,
getNearestParentPackageJSON,
deserializePackageJSON,
};

View File

@ -5,7 +5,9 @@ const {
} = primordials;
const { containsModuleSyntax } = internalBinding('contextify');
const { getNearestParentPackageJSONType } = internalBinding('modules');
const { getOptionValue } = require('internal/options');
const { checkPackageJSONIntegrity } = require('internal/modules/package_json_reader');
const path = require('path');
/**
@ -68,22 +70,27 @@ function shouldUseESMLoader(mainPath) {
if (mainPath && StringPrototypeEndsWith(mainPath, '.mjs')) { return true; }
if (!mainPath || StringPrototypeEndsWith(mainPath, '.cjs')) { return false; }
const { readPackageScope } = require('internal/modules/package_json_reader');
const pkg = readPackageScope(mainPath);
// No need to guard `pkg` as it can only be an object or `false`.
switch (pkg.data?.type) {
case 'module':
return true;
case 'commonjs':
return false;
default: { // No package.json or no `type` field.
const response = getNearestParentPackageJSONType(mainPath);
// No package.json or no `type` field.
if (response === undefined || response[0] === 'none') {
if (getOptionValue('--experimental-detect-module')) {
// If the first argument of `containsModuleSyntax` is undefined, it will read `mainPath` from the file system.
return containsModuleSyntax(undefined, mainPath);
}
return false;
}
}
// TODO(@anonrig): Do not return filePath and rawContent if experimental-policy is not used.
const {
0: type,
1: filePath,
2: rawContent,
} = response;
checkPackageJSONIntegrity(filePath, rawContent);
return type === 'module';
}
/**

View File

@ -112,6 +112,7 @@
'src/node_main_instance.cc',
'src/node_messaging.cc',
'src/node_metadata.cc',
'src/node_modules.cc',
'src/node_options.cc',
'src/node_os.cc',
'src/node_perf.cc',
@ -234,6 +235,7 @@
'src/node_messaging.h',
'src/node_metadata.h',
'src/node_mutex.h',
'src/node_modules.h',
'src/node_object_wrap.h',
'src/node_options.h',
'src/node_options-inl.h',

View File

@ -17,7 +17,8 @@ namespace node {
V(blob_binding_data, BlobBindingData) \
V(process_binding_data, process::BindingData) \
V(timers_binding_data, timers::BindingData) \
V(url_binding_data, url::BindingData)
V(url_binding_data, url::BindingData) \
V(modules_binding_data, modules::BindingData)
#define UNSERIALIZABLE_BINDING_TYPES(V) \
V(http2_binding_data, http2::BindingData) \

View File

@ -49,6 +49,7 @@
V(js_stream) \
V(js_udp_wrap) \
V(messaging) \
V(modules) \
V(module_wrap) \
V(mksnapshot) \
V(options) \

View File

@ -40,6 +40,7 @@ static_assert(static_cast<int>(NM_F_LINKED) ==
V(fs_dir) \
V(messaging) \
V(mksnapshot) \
V(modules) \
V(module_wrap) \
V(performance) \
V(process_methods) \

View File

@ -71,6 +71,7 @@ void AppendExceptionLine(Environment* env,
V(ERR_INVALID_ARG_TYPE, TypeError) \
V(ERR_INVALID_FILE_URL_HOST, TypeError) \
V(ERR_INVALID_FILE_URL_PATH, TypeError) \
V(ERR_INVALID_PACKAGE_CONFIG, Error) \
V(ERR_INVALID_OBJECT_DEFINE_PROPERTY, TypeError) \
V(ERR_INVALID_MODULE, Error) \
V(ERR_INVALID_STATE, Error) \

View File

@ -100,6 +100,7 @@ class ExternalReferenceRegistry {
V(messaging) \
V(mksnapshot) \
V(module_wrap) \
V(modules) \
V(options) \
V(os) \
V(performance) \

View File

@ -1038,68 +1038,6 @@ static void ExistsSync(const FunctionCallbackInfo<Value>& args) {
args.GetReturnValue().Set(err == 0);
}
// Used to speed up module loading. Returns an array [string, boolean]
static void InternalModuleReadJSON(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
Isolate* isolate = env->isolate();
uv_loop_t* loop = env->event_loop();
CHECK(args[0]->IsString());
node::Utf8Value path(isolate, args[0]);
THROW_IF_INSUFFICIENT_PERMISSIONS(
env, permission::PermissionScope::kFileSystemRead, path.ToStringView());
if (strlen(*path) != path.length()) {
return; // Contains a nul byte.
}
uv_fs_t open_req;
const int fd = uv_fs_open(loop, &open_req, *path, O_RDONLY, 0, nullptr);
uv_fs_req_cleanup(&open_req);
if (fd < 0) {
return;
}
auto defer_close = OnScopeLeave([fd, loop]() {
uv_fs_t close_req;
CHECK_EQ(0, uv_fs_close(loop, &close_req, fd, nullptr));
uv_fs_req_cleanup(&close_req);
});
const size_t kBlockSize = 32 << 10;
std::vector<char> chars;
int64_t offset = 0;
ssize_t numchars;
do {
const size_t start = chars.size();
chars.resize(start + kBlockSize);
uv_buf_t buf;
buf.base = &chars[start];
buf.len = kBlockSize;
uv_fs_t read_req;
numchars = uv_fs_read(loop, &read_req, fd, &buf, 1, offset, nullptr);
uv_fs_req_cleanup(&read_req);
if (numchars < 0) {
return;
}
offset += numchars;
} while (static_cast<size_t>(numchars) == kBlockSize);
size_t start = 0;
if (offset >= 3 && 0 == memcmp(chars.data(), "\xEF\xBB\xBF", 3)) {
start = 3; // Skip UTF-8 BOM.
}
const size_t size = offset - start;
args.GetReturnValue().Set(
String::NewFromUtf8(
isolate, &chars[start], v8::NewStringType::kNormal, size)
.ToLocalChecked());
}
// Used to speed up module loading. Returns 0 if the path refers to
// a file, 1 when it's a directory or < 0 on error (usually -ENOENT.)
// The speedup comes from not creating thousands of Stat and Error objects.
@ -3114,7 +3052,6 @@ static void CreatePerIsolateProperties(IsolateData* isolate_data,
SetMethod(isolate, target, "rmdir", RMDir);
SetMethod(isolate, target, "mkdir", MKDir);
SetMethod(isolate, target, "readdir", ReadDir);
SetMethod(isolate, target, "internalModuleReadJSON", InternalModuleReadJSON);
SetMethod(isolate, target, "internalModuleStat", InternalModuleStat);
SetMethod(isolate, target, "stat", Stat);
SetMethod(isolate, target, "lstat", LStat);
@ -3234,7 +3171,6 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
registry->Register(RMDir);
registry->Register(MKDir);
registry->Register(ReadDir);
registry->Register(InternalModuleReadJSON);
registry->Register(InternalModuleStat);
registry->Register(Stat);
registry->Register(LStat);

450
src/node_modules.cc Normal file
View File

@ -0,0 +1,450 @@
#include "node_modules.h"
#include <cstdio>
#include "base_object-inl.h"
#include "node_errors.h"
#include "node_external_reference.h"
#include "node_url.h"
#include "permission/permission.h"
#include "permission/permission_base.h"
#include "util-inl.h"
#include "v8-fast-api-calls.h"
#include "v8-function-callback.h"
#include "v8-primitive.h"
#include "v8-value.h"
#include "v8.h"
#include "simdjson.h"
namespace node {
namespace modules {
using v8::Array;
using v8::Context;
using v8::FunctionCallbackInfo;
using v8::HandleScope;
using v8::Isolate;
using v8::Local;
using v8::NewStringType;
using v8::Object;
using v8::ObjectTemplate;
using v8::Primitive;
using v8::String;
using v8::Undefined;
using v8::Value;
#ifdef __POSIX__
constexpr char kPathSeparator = '/';
constexpr std::string_view kNodeModules = "/node_modules";
#else
constexpr char kPathSeparator = '\\';
constexpr std::string_view kNodeModules = "\\node_modules";
#endif
void BindingData::MemoryInfo(MemoryTracker* tracker) const {
// Do nothing
}
BindingData::BindingData(Realm* realm,
v8::Local<v8::Object> object,
InternalFieldInfo* info)
: SnapshotableObject(realm, object, type_int) {}
bool BindingData::PrepareForSerialization(v8::Local<v8::Context> context,
v8::SnapshotCreator* creator) {
// Return true because we need to maintain the reference to the binding from
// JS land.
return true;
}
InternalFieldInfoBase* BindingData::Serialize(int index) {
DCHECK_IS_SNAPSHOT_SLOT(index);
InternalFieldInfo* info =
InternalFieldInfoBase::New<InternalFieldInfo>(type());
return info;
}
void BindingData::Deserialize(v8::Local<v8::Context> context,
v8::Local<v8::Object> holder,
int index,
InternalFieldInfoBase* info) {
DCHECK_IS_SNAPSHOT_SLOT(index);
HandleScope scope(context->GetIsolate());
Realm* realm = Realm::GetCurrent(context);
BindingData* binding = realm->AddBindingData<BindingData>(holder);
CHECK_NOT_NULL(binding);
}
Local<Array> BindingData::PackageConfig::Serialize(Realm* realm) const {
auto has_manifest = !realm->env()->options()->experimental_policy.empty();
auto isolate = realm->isolate();
const auto ToString = [isolate](std::string_view input) -> Local<Primitive> {
return String::NewFromUtf8(
isolate, input.data(), NewStringType::kNormal, input.size())
.ToLocalChecked();
};
Local<Value> values[7] = {
name.has_value() ? ToString(*name) : Undefined(isolate),
main.has_value() ? ToString(*main) : Undefined(isolate),
ToString(type),
imports.has_value() ? ToString(*imports) : Undefined(isolate),
exports.has_value() ? ToString(*exports) : Undefined(isolate),
has_manifest ? ToString(raw_json) : Undefined(isolate),
ToString(file_path),
};
return Array::New(isolate, values, 7);
}
const BindingData::PackageConfig* BindingData::GetPackageJSON(
Realm* realm, std::string_view path, ErrorContext* error_context) {
auto binding_data = realm->GetBindingData<BindingData>();
auto cache_entry = binding_data->package_configs_.find(path.data());
if (cache_entry != binding_data->package_configs_.end()) {
return &cache_entry->second;
}
PackageConfig package_config{};
package_config.file_path = path;
// No need to exclude BOM since simdjson will skip it.
if (ReadFileSync(&package_config.raw_json, path.data()) < 0) {
return nullptr;
}
simdjson::ondemand::document document;
simdjson::ondemand::object main_object;
simdjson::error_code error =
binding_data->json_parser.iterate(package_config.raw_json).get(document);
const auto throw_invalid_package_config = [error_context, path, realm]() {
if (error_context == nullptr) {
THROW_ERR_INVALID_PACKAGE_CONFIG(
realm->isolate(), "Invalid package config %s.", path.data());
} else if (error_context->base.has_value()) {
auto file_url = ada::parse(error_context->base.value());
CHECK(file_url);
auto file_path = url::FileURLToPath(realm->env(), *file_url);
CHECK(file_path.has_value());
THROW_ERR_INVALID_PACKAGE_CONFIG(
realm->isolate(),
"Invalid package config %s while importing \"%s\" from %s.",
path.data(),
error_context->specifier.c_str(),
file_path->c_str());
} else {
THROW_ERR_INVALID_PACKAGE_CONFIG(
realm->isolate(), "Invalid package config %s.", path.data());
}
return nullptr;
};
if (error || document.get_object().get(main_object)) {
return throw_invalid_package_config();
}
simdjson::ondemand::raw_json_string key;
simdjson::ondemand::value value;
std::string_view field_value;
simdjson::ondemand::json_type field_type;
for (auto field : main_object) {
// Throw error if getting key or value fails.
if (field.key().get(key) || field.value().get(value)) {
return throw_invalid_package_config();
}
if (key == "name") {
// Though there is a key "name" with a corresponding value,
// the value may not be a string or could be an invalid JSON string
if (value.get_string(package_config.name)) {
return throw_invalid_package_config();
}
} else if (key == "main") {
if (value.get_string(package_config.main)) {
return throw_invalid_package_config();
}
} else if (key == "exports") {
if (value.type().get(field_type)) {
return throw_invalid_package_config();
}
switch (field_type) {
case simdjson::ondemand::json_type::object:
case simdjson::ondemand::json_type::array: {
if (value.raw_json().get(field_value)) {
return throw_invalid_package_config();
}
package_config.exports = field_value;
break;
}
case simdjson::ondemand::json_type::string: {
if (value.get_string(package_config.exports)) {
return throw_invalid_package_config();
}
break;
}
default:
break;
}
} else if (key == "imports") {
if (value.type().get(field_type)) {
return throw_invalid_package_config();
}
switch (field_type) {
case simdjson::ondemand::json_type::array:
case simdjson::ondemand::json_type::object: {
if (value.raw_json().get(field_value)) {
return throw_invalid_package_config();
}
package_config.imports = field_value;
break;
}
case simdjson::ondemand::json_type::string: {
if (value.get_string(package_config.imports)) {
return throw_invalid_package_config();
}
break;
}
default:
break;
}
} else if (key == "type") {
if (value.get_string().get(field_value)) {
return throw_invalid_package_config();
}
// Only update type if it is "commonjs" or "module"
// The default value is "none" for backward compatibility.
if (field_value == "commonjs" || field_value == "module") {
package_config.type = field_value;
}
}
}
// package_config could be quite large, so we should move it instead of
// copying it.
auto cached = binding_data->package_configs_.insert(
{std::string(path), std::move(package_config)});
return &cached.first->second;
}
void BindingData::ReadPackageJSON(const FunctionCallbackInfo<Value>& args) {
CHECK_GE(args.Length(), 1); // path, [is_esm, base, specifier]
CHECK(args[0]->IsString()); // path
Realm* realm = Realm::GetCurrent(args);
auto isolate = realm->isolate();
Utf8Value path(isolate, args[0]);
bool is_esm = args[1]->IsTrue();
auto error_context = ErrorContext();
if (is_esm) {
CHECK(args[2]->IsUndefined() || args[2]->IsString()); // base
CHECK(args[3]->IsString()); // specifier
if (args[2]->IsString()) {
Utf8Value base_value(isolate, args[2]);
error_context.base = base_value.ToString();
}
Utf8Value specifier(isolate, args[3]);
error_context.specifier = specifier.ToString();
}
THROW_IF_INSUFFICIENT_PERMISSIONS(
realm->env(),
permission::PermissionScope::kFileSystemRead,
path.ToStringView());
auto package_json =
GetPackageJSON(realm, path.ToString(), is_esm ? &error_context : nullptr);
if (package_json == nullptr) {
return;
}
args.GetReturnValue().Set(package_json->Serialize(realm));
}
const BindingData::PackageConfig* BindingData::TraverseParent(
Realm* realm, std::string_view check_path) {
auto env = realm->env();
auto root_separator_index = check_path.find_first_of(kPathSeparator);
size_t separator_index = 0;
const bool is_permissions_enabled = env->permission()->enabled();
do {
separator_index = check_path.find_last_of(kPathSeparator);
check_path = check_path.substr(0, separator_index);
// We don't need to try "/"
if (check_path.empty()) {
break;
}
// Stop the search when the process doesn't have permissions
// to walk upwards
if (UNLIKELY(is_permissions_enabled &&
!env->permission()->is_granted(
permission::PermissionScope::kFileSystemRead,
std::string(check_path) + kPathSeparator))) {
return nullptr;
}
// Check if the path ends with `/node_modules`
if (check_path.size() >= kNodeModules.size() &&
std::equal(check_path.end() - kNodeModules.size(),
check_path.end(),
kNodeModules.begin())) {
return nullptr;
}
auto package_json = GetPackageJSON(
realm,
std::string(check_path) + kPathSeparator + "package.json",
nullptr);
if (package_json != nullptr) {
return package_json;
}
} while (separator_index > root_separator_index);
return nullptr;
}
void BindingData::GetNearestParentPackageJSON(
const v8::FunctionCallbackInfo<v8::Value>& args) {
CHECK_GE(args.Length(), 1);
CHECK(args[0]->IsString());
Realm* realm = Realm::GetCurrent(args);
Utf8Value path_value(realm->isolate(), args[0]);
auto package_json = TraverseParent(realm, path_value.ToStringView());
if (package_json != nullptr) {
args.GetReturnValue().Set(package_json->Serialize(realm));
}
}
void BindingData::GetNearestParentPackageJSONType(
const FunctionCallbackInfo<Value>& args) {
CHECK_GE(args.Length(), 1);
CHECK(args[0]->IsString());
Realm* realm = Realm::GetCurrent(args);
Utf8Value path(realm->isolate(), args[0]);
auto package_json = TraverseParent(realm, path.ToStringView());
if (package_json == nullptr) {
return;
}
Local<Value> values[3] = {
ToV8Value(realm->context(), package_json->type).ToLocalChecked(),
ToV8Value(realm->context(), package_json->file_path).ToLocalChecked(),
ToV8Value(realm->context(), package_json->raw_json).ToLocalChecked()};
args.GetReturnValue().Set(Array::New(realm->isolate(), values, 3));
}
void BindingData::GetPackageScopeConfig(
const FunctionCallbackInfo<Value>& args) {
CHECK_GE(args.Length(), 1);
CHECK(args[0]->IsString());
Realm* realm = Realm::GetCurrent(args);
Utf8Value resolved(realm->isolate(), args[0]);
auto package_json_url_base = ada::parse(resolved.ToStringView());
if (!package_json_url_base) {
url::ThrowInvalidURL(realm->env(), resolved.ToStringView(), std::nullopt);
return;
}
auto package_json_url =
ada::parse("./package.json", &package_json_url_base.value());
if (!package_json_url) {
url::ThrowInvalidURL(realm->env(), "./package.json", resolved.ToString());
return;
}
std::string_view node_modules_package_path = "node_modules/package.json";
auto error_context = ErrorContext();
error_context.is_esm = true;
// TODO(@anonrig): Rewrite this function and avoid calling URL parser.
while (true) {
auto pathname = package_json_url->get_pathname();
if (pathname.size() >= node_modules_package_path.size() &&
pathname.compare(pathname.size() - node_modules_package_path.size(),
node_modules_package_path.size(),
node_modules_package_path) == 0) {
break;
}
auto file_url = url::FileURLToPath(realm->env(), *package_json_url);
CHECK(file_url);
error_context.specifier = resolved.ToString();
auto package_json = GetPackageJSON(realm, *file_url, &error_context);
if (package_json != nullptr) {
return args.GetReturnValue().Set(package_json->Serialize(realm));
}
auto last_href = std::string(package_json_url->get_href());
auto last_pathname = std::string(package_json_url->get_pathname());
package_json_url = ada::parse("../package.json", &package_json_url.value());
if (!package_json_url) {
url::ThrowInvalidURL(realm->env(), "../package.json", last_href);
return;
}
// Terminates at root where ../package.json equals ../../package.json
// (can't just check "/package.json" for Windows support).
if (package_json_url->get_pathname() == last_pathname) {
break;
}
}
auto package_json_url_as_path =
url::FileURLToPath(realm->env(), *package_json_url);
CHECK(package_json_url_as_path);
return args.GetReturnValue().Set(
String::NewFromUtf8(realm->isolate(),
package_json_url_as_path->c_str(),
NewStringType::kNormal,
package_json_url_as_path->size())
.ToLocalChecked());
}
void BindingData::CreatePerIsolateProperties(IsolateData* isolate_data,
Local<ObjectTemplate> target) {
Isolate* isolate = isolate_data->isolate();
SetMethod(isolate, target, "readPackageJSON", ReadPackageJSON);
SetMethod(isolate,
target,
"getNearestParentPackageJSONType",
GetNearestParentPackageJSONType);
SetMethod(isolate,
target,
"getNearestParentPackageJSON",
GetNearestParentPackageJSON);
SetMethod(isolate, target, "getPackageScopeConfig", GetPackageScopeConfig);
}
void BindingData::CreatePerContextProperties(Local<Object> target,
Local<Value> unused,
Local<Context> context,
void* priv) {
Realm* realm = Realm::GetCurrent(context);
realm->AddBindingData<BindingData>(target);
}
void BindingData::RegisterExternalReferences(
ExternalReferenceRegistry* registry) {
registry->Register(ReadPackageJSON);
registry->Register(GetNearestParentPackageJSONType);
registry->Register(GetNearestParentPackageJSON);
registry->Register(GetPackageScopeConfig);
}
} // namespace modules
} // namespace node
NODE_BINDING_CONTEXT_AWARE_INTERNAL(
modules, node::modules::BindingData::CreatePerContextProperties)
NODE_BINDING_PER_ISOLATE_INIT(
modules, node::modules::BindingData::CreatePerIsolateProperties)
NODE_BINDING_EXTERNAL_REFERENCE(
modules, node::modules::BindingData::RegisterExternalReferences)

89
src/node_modules.h Normal file
View File

@ -0,0 +1,89 @@
#ifndef SRC_NODE_MODULES_H_
#define SRC_NODE_MODULES_H_
#include "v8-function-callback.h"
#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS
#include "node.h"
#include "node_snapshotable.h"
#include "simdjson.h"
#include "util.h"
#include "v8-fast-api-calls.h"
#include "v8.h"
#include <optional>
#include <string>
#include <string_view>
#include <unordered_map>
namespace node {
class ExternalReferenceRegistry;
namespace modules {
class BindingData : public SnapshotableObject {
public:
using InternalFieldInfo = InternalFieldInfoBase;
struct PackageConfig {
std::string file_path;
std::optional<std::string> name;
std::optional<std::string> main;
std::string type = "none";
std::optional<std::string> exports;
std::optional<std::string> imports;
std::string raw_json;
v8::Local<v8::Array> Serialize(Realm* realm) const;
};
struct ErrorContext {
std::optional<std::string> base;
std::string specifier;
bool is_esm;
};
BindingData(Realm* realm,
v8::Local<v8::Object> obj,
InternalFieldInfo* info = nullptr);
SERIALIZABLE_OBJECT_METHODS()
SET_BINDING_ID(modules_binding_data)
void MemoryInfo(MemoryTracker* tracker) const override;
SET_SELF_SIZE(BindingData)
SET_MEMORY_INFO_NAME(BindingData)
static void ReadPackageJSON(const v8::FunctionCallbackInfo<v8::Value>& args);
static void GetNearestParentPackageJSON(
const v8::FunctionCallbackInfo<v8::Value>& args);
static void GetNearestParentPackageJSONType(
const v8::FunctionCallbackInfo<v8::Value>& args);
static void GetPackageScopeConfig(
const v8::FunctionCallbackInfo<v8::Value>& args);
static void CreatePerIsolateProperties(IsolateData* isolate_data,
v8::Local<v8::ObjectTemplate> ctor);
static void CreatePerContextProperties(v8::Local<v8::Object> target,
v8::Local<v8::Value> unused,
v8::Local<v8::Context> context,
void* priv);
static void RegisterExternalReferences(ExternalReferenceRegistry* registry);
private:
std::unordered_map<std::string, PackageConfig> package_configs_;
simdjson::ondemand::parser json_parser;
// returns null on error
static const PackageConfig* GetPackageJSON(
Realm* realm,
std::string_view path,
ErrorContext* error_context = nullptr);
static const PackageConfig* TraverseParent(Realm* realm,
std::string_view check_path);
};
} // namespace modules
} // namespace node
#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS
#endif // SRC_NODE_MODULES_H_

View File

@ -19,6 +19,7 @@
#include "node_internals.h"
#include "node_main_instance.h"
#include "node_metadata.h"
#include "node_modules.h"
#include "node_process.h"
#include "node_snapshot_builder.h"
#include "node_url.h"

View File

@ -229,35 +229,6 @@ void BindingData::Format(const FunctionCallbackInfo<Value>& args) {
.ToLocalChecked());
}
void BindingData::ThrowInvalidURL(node::Environment* env,
std::string_view input,
std::optional<std::string> base) {
Local<Value> err = ERR_INVALID_URL(env->isolate(), "Invalid URL");
DCHECK(err->IsObject());
auto err_object = err.As<Object>();
USE(err_object->Set(env->context(),
env->input_string(),
v8::String::NewFromUtf8(env->isolate(),
input.data(),
v8::NewStringType::kNormal,
input.size())
.ToLocalChecked()));
if (base.has_value()) {
USE(err_object->Set(env->context(),
env->base_string(),
v8::String::NewFromUtf8(env->isolate(),
base.value().c_str(),
v8::NewStringType::kNormal,
base.value().size())
.ToLocalChecked()));
}
env->isolate()->ThrowException(err);
}
void BindingData::Parse(const FunctionCallbackInfo<Value>& args) {
CHECK_GE(args.Length(), 1);
CHECK(args[0]->IsString()); // input
@ -419,6 +390,35 @@ void BindingData::RegisterExternalReferences(
}
}
void ThrowInvalidURL(node::Environment* env,
std::string_view input,
std::optional<std::string> base) {
Local<Value> err = ERR_INVALID_URL(env->isolate(), "Invalid URL");
DCHECK(err->IsObject());
auto err_object = err.As<Object>();
USE(err_object->Set(env->context(),
env->input_string(),
v8::String::NewFromUtf8(env->isolate(),
input.data(),
v8::NewStringType::kNormal,
input.size())
.ToLocalChecked()));
if (base.has_value()) {
USE(err_object->Set(env->context(),
env->base_string(),
v8::String::NewFromUtf8(env->isolate(),
base.value().c_str(),
v8::NewStringType::kNormal,
base.value().size())
.ToLocalChecked()));
}
env->isolate()->ThrowException(err);
}
std::string FromFilePath(std::string_view file_path) {
// Avoid unnecessary allocations.
size_t pos = file_path.empty() ? std::string_view::npos : file_path.find('%');

View File

@ -77,11 +77,11 @@ class BindingData : public SnapshotableObject {
const ada::scheme::type type);
static v8::CFunction fast_can_parse_methods_[];
static void ThrowInvalidURL(Environment* env,
std::string_view input,
std::optional<std::string> base);
};
void ThrowInvalidURL(Environment* env,
std::string_view input,
std::optional<std::string> base);
std::string FromFilePath(std::string_view file_path);
std::optional<std::string> FileURLToPath(Environment* env,
const ada::url_aggregator& file_url);

View File

@ -1,6 +1,6 @@
'use strict';
const { checkoutEOL, spawnPromisified } = require('../common');
const { spawnPromisified } = require('../common');
const fixtures = require('../common/fixtures.js');
const assert = require('node:assert');
const { execPath } = require('node:process');
@ -14,12 +14,10 @@ describe('ESM: Package.json', { concurrency: true }, () => {
const { code, signal, stderr } = await spawnPromisified(execPath, [entry]);
assert.ok(stderr.includes('code: \'ERR_INVALID_PACKAGE_CONFIG\''), stderr);
assert.ok(
stderr.includes(
`[ERR_INVALID_PACKAGE_CONFIG]: Invalid package config ${invalidJson} ` +
`while importing "invalid-pjson" from ${entry}. ` +
"Expected ':' after property name in JSON at position " +
`${12 + checkoutEOL.length * 2}`
`Invalid package config ${invalidJson} while importing "invalid-pjson" from ${entry}.`
),
stderr
);

View File

@ -1,3 +1,3 @@
{
"exports": 42
"exports": {}
}

View File

@ -11,6 +11,7 @@ const assert = require('assert');
const expectedModules = new Set([
'Internal Binding builtins',
'Internal Binding encoding_binding',
'Internal Binding modules',
'Internal Binding errors',
'Internal Binding util',
'NativeModule internal/errors',

View File

@ -1,29 +0,0 @@
// Flags: --expose-internals
'use strict';
require('../common');
const fixtures = require('../common/fixtures');
const { internalBinding } = require('internal/test/binding');
const { filterOwnProperties } = require('internal/util');
const { internalModuleReadJSON } = internalBinding('fs');
const { readFileSync } = require('fs');
const { strictEqual, deepStrictEqual } = require('assert');
{
strictEqual(internalModuleReadJSON('nosuchfile'), undefined);
}
{
strictEqual(internalModuleReadJSON(fixtures.path('empty.txt')), '');
}
{
strictEqual(internalModuleReadJSON(fixtures.path('empty-with-bom.txt')), '');
}
{
const filename = fixtures.path('require-bin/package.json');
const returnValue = JSON.parse(internalModuleReadJSON(filename));
const file = JSON.parse(readFileSync(filename, 'utf-8'));
const expectedValue = filterOwnProperties(file, ['name', 'main', 'exports', 'imports', 'type']);
deepStrictEqual({
__proto__: null,
...returnValue,
}, expectedValue);
}

View File

@ -14,6 +14,7 @@ import {TypesBinding} from "./internalBinding/types";
import {URLBinding} from "./internalBinding/url";
import {UtilBinding} from "./internalBinding/util";
import {WorkerBinding} from "./internalBinding/worker";
import {ModulesBinding} from "./internalBinding/modules";
declare type TypedArray =
| Uint8Array
@ -36,6 +37,7 @@ interface InternalBindingMap {
fs: FsBinding;
http_parser: HttpParserBinding;
messaging: MessagingBinding;
modules: ModulesBinding;
options: OptionsBinding;
os: OSBinding;
serdes: SerdesBinding;

View File

@ -111,7 +111,6 @@ declare namespace InternalFSBinding {
function futimes(fd: number, atime: number, mtime: number): void;
function futimes(fd: number, atime: number, mtime: number, usePromises: typeof kUsePromises): Promise<void>;
function internalModuleReadJSON(path: string): [] | [string, boolean];
function internalModuleStat(path: string): number;
function lchown(path: string, uid: number, gid: number, req: FSReqCallback): void;

29
typings/internalBinding/modules.d.ts vendored Normal file
View File

@ -0,0 +1,29 @@
export type PackageType = 'commonjs' | 'module' | 'none'
export type PackageConfig = {
pjsonPath: string
exists: boolean
name?: string
main?: any
type: PackageType
exports?: string | string[] | Record<string, unknown>
imports?: string | string[] | Record<string, unknown>
}
export type SerializedPackageConfig = [
PackageConfig['name'],
PackageConfig['main'],
PackageConfig['type'],
string | undefined, // exports
string | undefined, // imports
string | undefined, // raw json available for experimental policy
]
export interface ModulesBinding {
readPackageJSON(path: string): SerializedPackageConfig | undefined;
getNearestParentPackageJSON(path: string): PackageConfig | undefined
getNearestParentPackageJSONType(path: string): [
PackageConfig['type'],
string, // package.json path
string, // raw content
]
getPackageScopeConfig(path: string): SerializedPackageConfig | undefined
}