1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Teach moc to output a Make-style depfile

Browse Source 
If moc is invoked with the --output-dep-file option, it will generate
a "moc_<source_file_name>.d" dep file which contains dependency
entries that can be consumed by a Makefile / Ninja build system.

This is useful for build tools (like CMake) to know when moc should be
re-ran.
In the future, it might also be useful for ccache (teach ccache not to
re-run moc when not necessary).

The dependency list contains: the original source file, the passed
--include files (like moc_predefs.h), the include files that
were discovered while preprocessing the source file, and the plugin
metadata json files listed in Q_PLUGIN_METADATA macros.

The file paths are encoded using QFile::encodeName, so using the local
8-bit encoding.

The paths are also escaped (so ' ' replaced by '\ ', '$' by '$$',
etc) according to the Make-style rules as described in
clang's dep file generator
https://github.com/llvm/llvm-project/blob/release/9.x/clang/lib/Frontend/DependencyFile.cpp#L233

For reference, the equivalent Ninja depfile parser source code can be
found at
https://github.com/ninja-build/ninja/blob/v1.9.0/src/depfile_parser.in.cc#L37

Additional options that can be passed:
--dep-file-path - to change the location where the dep file should be
generated.
--dep-file-rule-name - to change the rule name (first line) of the
dep file (useful when no -o option is specified, so output goes to
stdout).

Encoding story.
Note that moc doesn't handle non-local-8-bit characters properly when
processing include directives at the preprocessor step. Specifically
the content of the main input file is read as a raw byte array (which
can be UTF-8 encoded) and then each include directive is resolved via
Preprocessor::resolveInclude(), which calls QString::fromLocal8Bit().

Because moc uses the QtBootstrap library, only a limited set of codecs
are available: various UTF 8 / 16 / 32 codecs and
QLatin1Codec (ISO-8859-15).

This means that on Windows, if the source input file is UTF-8 encoded,
and contains include names with UTF-8 characters (like an emoji or any
character >= 127 that is not in the QLatin1 codec), moc will fail to
resolve and process that include, and thus no dep file entry will be
created either.

On macOS / QNX / WASM the main locale is UTF-8, so file content
and paths will be processed correctly (hardcoded via QT_LOCALE_IS_UTF8
in src/corelib/codecs/qtextcodec_p.h).

On Linux it will depend on the current locale / encoding set,
and if that encoding is one of the ones supported above. UTF-8 should
work fine.

[ChangeLog][QtCore][moc] moc can now output a ".d" dep file that can
be consumed by other build systems.

Task-number: QTBUG-74521
Task-number: QTBUG-76598
Change-Id: I5585631ff1bbbae4e2875cade9cb6c20ed018c0a
Reviewed-by: Leander Beernaert <leander.beernaert@qt.io>
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
This commit is contained in:
Alexandru Croitor 2019-12-10 15:01:11 +01:00
parent 18f22fea7c
commit cf3d4cf3c3
3 changed files with 133 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
src/tools/moc/main.cpp
View File

@ -175,6 +175,49 @@ static QStringList argumentsFromCommandLineAndFile(const QStringList &arguments)
return allArguments;
}
// Escape characters in given path. Dependency paths are Make-style, not NMake/Jom style.
// The paths can also be consumed by Ninja.
// "$" replaced by "$$"
// "#" replaced by "\#"
// " " replaced by "\ "
// "\#" replaced by "\\#"
// "\ " replaced by "\\\ "
//
// The escape rules are according to what clang / llvm escapes when generating a Make-style
// dependency file.
// Is a template function, because input param can be either a QString or a QByteArray.
template <typename T> struct CharType;
template <> struct CharType<QString> { using type = QLatin1Char; };
template <> struct CharType<QByteArray> { using type = char; };
template <typename StringType>
StringType escapeDependencyPath(const StringType &path)
{
using CT = typename CharType<StringType>::type;
StringType escapedPath;
int size = path.size();
escapedPath.reserve(size);
for (int i = 0; i < size; ++i) {
if (path[i] == CT('$')) {
escapedPath.append(CT('$'));
} else if (path[i] == CT('#')) {
escapedPath.append(CT('\\'));
} else if (path[i] == CT(' ')) {
escapedPath.append(CT('\\'));
int backwards_it = i - 1;
while (backwards_it > 0 && path[backwards_it] == CT('\\')) {
escapedPath.append(CT('\\'));
--backwards_it;
}
}
escapedPath.append(path[i]);
}
return escapedPath;
}
QByteArray escapeAndEncodeDependencyPath(const QString &path)
{
return QFile::encodeName(escapeDependencyPath(path));
}
int runMoc(int argc, char **argv)
{
@ -308,6 +351,22 @@ int runMoc(int argc, char **argv)
collectOption.setDescription(QStringLiteral("Instead of processing C++ code, collect previously generated JSON output into a single file."));
parser.addOption(collectOption);
QCommandLineOption depFileOption(QStringLiteral("output-dep-file"));
depFileOption.setDescription(
QStringLiteral("Output a Make-style dep file for build system consumption."));
parser.addOption(depFileOption);
QCommandLineOption depFilePathOption(QStringLiteral("dep-file-path"));
depFilePathOption.setDescription(QStringLiteral("Path where to write the dep file."));
depFilePathOption.setValueName(QStringLiteral("file"));
parser.addOption(depFilePathOption);
QCommandLineOption depFileRuleNameOption(QStringLiteral("dep-file-rule-name"));
depFileRuleNameOption.setDescription(
QStringLiteral("The rule name (first line) of the dep file."));
depFileRuleNameOption.setValueName(QStringLiteral("rule name"));
parser.addOption(depFileRuleNameOption);
parser.addPositionalArgument(QStringLiteral("[header-file]"),
QStringLiteral("Header file to read from, otherwise stdin."));
parser.addPositionalArgument(QStringLiteral("[@option-file]"),
@ -476,6 +535,7 @@ int runMoc(int argc, char **argv)
// 1. preprocess
const auto includeFiles = parser.values(includeOption);
QStringList validIncludesFiles;
for (const QString &includeName : includeFiles) {
QByteArray rawName = pp.resolveInclude(QFile::encodeName(includeName), moc.filename);
if (rawName.isEmpty()) {
@ -488,6 +548,7 @@ int runMoc(int argc, char **argv)
moc.symbols += Symbol(0, MOC_INCLUDE_BEGIN, rawName);
moc.symbols += pp.preprocessed(rawName, &f);
moc.symbols += Symbol(0, MOC_INCLUDE_END, rawName);
validIncludesFiles.append(includeName);
} else {
fprintf(stderr, "Warning: Cannot open %s included by moc file %s: %s\n",
rawName.constData(),
@ -507,6 +568,7 @@ int runMoc(int argc, char **argv)
QScopedPointer<FILE, ScopedPointerFileCloser> jsonOutput;
bool outputToFile = true;
if (output.size()) { // output file specified
#if defined(_MSC_VER)
if (_wfopen_s(&out, reinterpret_cast<const wchar_t *>(output.utf16()), L"w") != 0)
@ -535,6 +597,7 @@ int runMoc(int argc, char **argv)
}
} else { // use stdout
out = stdout;
outputToFile = false;
}
if (pp.preprocessOnly) {
@ -549,6 +612,74 @@ int runMoc(int argc, char **argv)
if (output.size())
fclose(out);
if (parser.isSet(depFileOption)) {
// 4. write a Make-style dependency file (can also be consumed by Ninja).
QString depOutputFileName;
QString depRuleName = output;
if (parser.isSet(depFileRuleNameOption))
depRuleName = parser.value(depFileRuleNameOption);
if (parser.isSet(depFilePathOption)) {
depOutputFileName = parser.value(depFilePathOption);
} else if (outputToFile) {
depOutputFileName = output + QLatin1String(".d");
} else {
fprintf(stderr, "moc: Writing to stdout, but no depfile path specified.\n");
}
QScopedPointer<FILE, ScopedPointerFileCloser> depFileHandle;
FILE *depFileHandleRaw;
#if defined(_MSC_VER)
if (_wfopen_s(&depFileHandleRaw,
reinterpret_cast<const wchar_t *>(depOutputFileName.utf16()), L"w") != 0)
#else
depFileHandleRaw = fopen(QFile::encodeName(depOutputFileName).constData(), "w");
if (!depFileHandleRaw)
#endif
fprintf(stderr, "moc: Cannot create dep output file '%s'. %s\n",
QFile::encodeName(depOutputFileName).constData(),
strerror(errno));
depFileHandle.reset(depFileHandleRaw);
if (!depFileHandle.isNull()) {
// First line is the path to the generated file.
fprintf(depFileHandle.data(), "%s: ",
escapeAndEncodeDependencyPath(depRuleName).constData());
QByteArrayList dependencies;
// If there's an input file, it's the first dependency.
if (!filename.isEmpty()) {
dependencies.append(escapeAndEncodeDependencyPath(filename).constData());
}
// Additional passed-in includes are dependencies (like moc_predefs.h).
for (const QString &includeName : validIncludesFiles) {
dependencies.append(escapeAndEncodeDependencyPath(includeName).constData());
}
// Plugin metadata json files discovered via Q_PLUGIN_METADATA macros are also
// dependencies.
for (const QString &pluginMetadataFile : moc.parsedPluginMetadataFiles) {
dependencies.append(escapeAndEncodeDependencyPath(pluginMetadataFile).constData());
}
// All pre-processed includes are dependnecies.
// Sort the entries for easier human consumption.
auto includeList = pp.preprocessedIncludes.values();
std::sort(includeList.begin(), includeList.end());
for (QByteArray &includeName : includeList) {
dependencies.append(escapeDependencyPath(includeName));
}
// Join dependencies, output them, and output a final new line.
const auto dependenciesJoined = dependencies.join(QByteArrayLiteral(" \\\n "));
fprintf(depFileHandle.data(), "%s\n", dependenciesJoined.constData());
}
}
return 0;
}

1
src/tools/moc/moc.cpp
View File

@ -1382,6 +1382,7 @@ void Moc::parsePluginData(ClassDef *def)
error(msg.constData());
return;
}
parsedPluginMetadataFiles.append(fi.canonicalFilePath());
metaData = file.readAll();
}
}

1
src/tools/moc/moc.h
View File

@ -223,6 +223,7 @@ public:
QHash<QByteArray, QByteArray> knownQObjectClasses;
QHash<QByteArray, QByteArray> knownGadgets;
QMap<QString, QJsonArray> metaArgs;
QVector<QString> parsedPluginMetadataFiles;
void parse();
void generate(FILE *out, FILE *jsonOutput);