Windeployqt: add options to deploy/block plugins

Some plugin types are pulled in by default by certain modules.
Give users the option to add/skip plugins and/or their types.

[ChangeLog][Tools][Windeployqt] Windeployqt now has options that allow for custom plugin deployment. Users can include or exclude them, either individually, or by type.

Fixes: QTBUG-117910
Change-Id: I85235783dcd814396f184912269cd5976717b2dd
Reviewed-by: Oliver Wolff <oliver.wolff@qt.io>
(cherry picked from commit d53c0d721f111d53ea95eb8914ad88560a00feaa)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

src/tools/windeployqt/CMakeLists.txt

@@ -14,6 +14,7 @@ qt_internal_add_tool(${target_name}
     SOURCES
         qmlutils.cpp qmlutils.h
         qtmoduleinfo.cpp qtmoduleinfo.h
+        qtplugininfo.cpp qtplugininfo.h
         utils.cpp utils.h
         main.cpp
     DEFINES

src/tools/windeployqt/main.cpp

@@ -4,6 +4,7 @@
 #include "utils.h"
 #include "qmlutils.h"
 #include "qtmoduleinfo.h"
+#include "qtplugininfo.h"
 
 #include <QtCore/QCommandLineOption>
 #include <QtCore/QCommandLineParser>
@@ -114,6 +115,21 @@ static QByteArray formatQtModules(const ModuleBitset &mask, bool option = false)
     return result;
 }
 
+static QString formatQtPlugins(const PluginInformation &pluginInfo)
+{
+    QString result(u'\n');
+    for (const auto &pair : pluginInfo.typeMap()) {
+        result += pair.first;
+        result += u": \n";
+        for (const QString &plugin : pair.second) {
+            result += u"    ";
+            result += plugin;
+            result += u'\n';
+        }
+    }
+    return result;
+}
+
 static Platform platformFromMkSpec(const QString &xSpec)
 {
     if (xSpec.startsWith("win32-"_L1)) {
@@ -163,8 +179,8 @@ struct Options {
     bool translations = true;
     bool systemD3dCompiler = true;
     bool compilerRunTime = false;
-    QStringList disabledPluginTypes;
     bool softwareRasterizer = true;
+    PluginLists pluginSelections;
     Platform platform = WindowsDesktopMsvc;
     ModuleBitset additionalLibraries;
     ModuleBitset disabledLibraries;
@@ -385,6 +401,21 @@ static inline int parseArguments(const QStringList &arguments, QCommandLineParse
                                          QStringLiteral("plugin types"));
     parser->addOption(skipPluginTypesOption);
 
+    QCommandLineOption addPluginTypesOption(QStringLiteral("add-plugin-types"),
+                                            QStringLiteral("A comma-separated list of plugin types that will be added to deployment (imageformats,iconengines)"),
+                                            QStringLiteral("plugin types"));
+    parser->addOption(addPluginTypesOption);
+
+    QCommandLineOption includePluginsOption(QStringLiteral("include-plugins"),
+                                            QStringLiteral("A comma-separated list of individual plugins that will be added to deployment (scene2d,qjpeg)"),
+                                            QStringLiteral("plugins"));
+    parser->addOption(includePluginsOption);
+
+    QCommandLineOption excludePluginsOption(QStringLiteral("exclude-plugins"),
+                                            QStringLiteral("A comma-separated list of individual plugins that will not be deployed (qsvg,qpdf)"),
+                                            QStringLiteral("plugins"));
+    parser->addOption(excludePluginsOption);
+
     QCommandLineOption noLibraryOption(QStringLiteral("no-libraries"),
                                        QStringLiteral("Skip library deployment."));
     parser->addOption(noLibraryOption);
@@ -514,7 +545,16 @@ static inline int parseArguments(const QStringList &arguments, QCommandLineParse
     }
 
     if (parser->isSet(skipPluginTypesOption))
-        options->disabledPluginTypes = parser->value(skipPluginTypesOption).split(u',');
+        options->pluginSelections.disabledPluginTypes = parser->value(skipPluginTypesOption).split(u',');
+
+    if (parser->isSet(addPluginTypesOption))
+        options->pluginSelections.enabledPluginTypes = parser->value(addPluginTypesOption).split(u',');
+
+    if (parser->isSet(includePluginsOption))
+        options->pluginSelections.includedPlugins = parser->value(includePluginsOption).split(u',');
+
+    if (parser->isSet(excludePluginsOption))
+        options->pluginSelections.excludedPlugins = parser->value(excludePluginsOption).split(u',');
 
     if (parser->isSet(releaseWithDebugInfoOption))
         std::wcerr << "Warning: " << releaseWithDebugInfoOption.names().first() << " is obsolete.";
@@ -667,7 +707,7 @@ static inline QString lineBreak(QString s)
     return s;
 }
 
-static inline QString helpText(const QCommandLineParser &p)
+static inline QString helpText(const QCommandLineParser &p, const PluginInformation &pluginInfo)
 {
     QString result = p.helpText();
     // Replace the default-generated text which is too long by a short summary
@@ -686,7 +726,18 @@ static inline QString helpText(const QCommandLineParser &p)
         "the name prepended by --no- (--no-xml). Available libraries:\n"_L1;
     ModuleBitset mask;
     moduleHelp += lineBreak(QString::fromLatin1(formatQtModules(mask.set(), true)));
-    moduleHelp += u'\n';
+    moduleHelp += u"\n\n";
+    moduleHelp +=
+            u"Qt plugins can be included or excluded individually or by type.\n"
+            u"To deploy or block plugins individually, use the --include-plugins\n"
+            u"and --exclude-plugins options (--include-plugins qjpeg,qsvgicon)\n"
+            u"You can also use the --skip-plugin-types or --add-plugin-types to\n"
+            u"achieve similar results with entire plugin groups, like imageformats, e.g.\n"
+            u"(--add-plugin-types imageformats,iconengines). Exclusion always takes\n"
+            u"precedence over inclusion, and types take precedence over specific plugins.\n"
+            u"For example, including qjpeg, but skipping imageformats, will NOT deploy qjpeg.\n"
+            u"\nDetected available plugins:\n";
+    moduleHelp += formatQtPlugins(pluginInfo);
     result.replace(moduleStart, argumentsStart - moduleStart, moduleHelp);
     return result;
 }
@@ -860,15 +911,16 @@ static qint64 qtModule(QString module, const QString &infix)
 }
 
 // Return the path if a plugin is to be deployed
-static QString deployPlugin(const QString &plugin, const QDir &subDir,
+static QString deployPlugin(const QString &plugin, const QDir &subDir, const bool dueToModule,
-                            ModuleBitset *usedQtModules, const ModuleBitset &disabledQtModules,
+                            const DebugMatchMode &debugMatchMode, ModuleBitset *usedQtModules,
-                            const QStringList &disabledPluginTypes,
+                            const ModuleBitset &disabledQtModules,
-                            const QString &libraryLocation, const QString &infix,
+                            const PluginLists &pluginSelections, const QString &libraryLocation,
-                            Platform platform, bool deployInsightTrackerPlugin)
+                            const QString &infix, Platform platform,
+                            bool deployInsightTrackerPlugin)
 {
     const QString subDirName = subDir.dirName();
     // Filter out disabled plugins
-    if (disabledPluginTypes.contains(subDirName)) {
+    if (pluginSelections.disabledPluginTypes.contains(subDirName)) {
         std::wcout << "Skipping plugin " << plugin << " due to skipped plugin type " << subDirName << '\n';
         return {};
     }
@@ -879,7 +931,28 @@ static QString deployPlugin(const QString &plugin, const QDir &subDir,
         return {};
     }
 
+    const int dotIndex = plugin.lastIndexOf(u'.');
+    // Strip the .dll from the name, and an additional 'd' if it's a debug library with the 'd'
+    // suffix
+    const int stripIndex = debugMatchMode == MatchDebug && platformHasDebugSuffix(platform)
+            ? dotIndex - 1
+            : dotIndex;
+    const QString pluginName = plugin.first(stripIndex);
+
+    if (pluginSelections.excludedPlugins.contains(pluginName)) {
+        std::wcout << "Skipping plugin " << plugin << " due to exclusion option" << '\n';
+        return {};
+    }
+
     const QString pluginPath = subDir.absoluteFilePath(plugin);
+
+    // If dueToModule is false, check if the user included the plugin or the entire type. In the
+    // former's case, only deploy said plugin and not all plugins of that type.
+    const bool requiresPlugin = pluginSelections.includedPlugins.contains(pluginName)
+            || pluginSelections.enabledPluginTypes.contains(subDirName);
+    if (!dueToModule && !requiresPlugin)
+        return {};
+
     // Deploy QUiTools plugins as is without further dependency checking.
     // The user needs to ensure all required libraries are present (would
     // otherwise pull QtWebEngine for its plugin).
@@ -923,12 +996,22 @@ static QString deployPlugin(const QString &plugin, const QDir &subDir,
     return pluginPath;
 }
 
+static bool needsPluginType(const QString &subDirName, const PluginInformation &pluginInfo,
+                            const PluginLists &pluginSelections)
+{
+    bool needsTypeForPlugin = false;
+    for (const QString &plugin: pluginSelections.includedPlugins) {
+        if (pluginInfo.isTypeForPlugin(subDirName, plugin))
+            needsTypeForPlugin = true;
+    }
+    return (pluginSelections.enabledPluginTypes.contains(subDirName) || needsTypeForPlugin);
+}
+
 QStringList findQtPlugins(ModuleBitset *usedQtModules, const ModuleBitset &disabledQtModules,
-                          const QStringList &disabledPluginTypes,
+                          const PluginInformation &pluginInfo, const PluginLists &pluginSelections,
                           const QString &qtPluginsDirName, const QString &libraryLocation,
-                          const QString &infix,
+                          const QString &infix, DebugMatchMode debugMatchModeIn, Platform platform,
-                          DebugMatchMode debugMatchModeIn, Platform platform, QString *platformPlugin,
+                          QString *platformPlugin, bool deployInsightTrackerPlugin)
-                          bool deployInsightTrackerPlugin)
 {
     if (qtPluginsDirName.isEmpty())
         return QStringList();
@@ -944,7 +1027,8 @@ QStringList findQtPlugins(ModuleBitset *usedQtModules, const ModuleBitset &disab
             }
             continue;
         }
-        if (usedQtModules->test(module)) {
+        const bool dueToModule = usedQtModules->test(module);
+        if (dueToModule || needsPluginType(subDirName, pluginInfo, pluginSelections)) {
             const DebugMatchMode debugMatchMode = (module == QtWebEngineCoreModuleId)
                 ? MatchDebugOrRelease // QTBUG-44331: Debug detection does not work for webengine, deploy all.
                 : debugMatchModeIn;
@@ -960,12 +1044,13 @@ QStringList findQtPlugins(ModuleBitset *usedQtModules, const ModuleBitset &disab
             } else {
                 filter = u"*"_s;
             }
-            const QStringList plugins = findSharedLibraries(subDir, platform, debugMatchMode, filter);
+            const QStringList plugins =
+                    findSharedLibraries(subDir, platform, debugMatchMode, filter);
             for (const QString &plugin : plugins) {
                 const QString pluginPath =
-                    deployPlugin(plugin, subDir, usedQtModules, disabledQtModules,
+                        deployPlugin(plugin, subDir, dueToModule, debugMatchMode, usedQtModules,
-                                 disabledPluginTypes, libraryLocation, infix, platform,
+                                     disabledQtModules, pluginSelections, libraryLocation, infix,
-                                 deployInsightTrackerPlugin);
+                                     platform, deployInsightTrackerPlugin);
                 if (!pluginPath.isEmpty()) {
                     if (isPlatformPlugin)
                         *platformPlugin = subDir.absoluteFilePath(plugin);
@@ -1238,7 +1323,7 @@ static QString getIcuVersion(const QString &libName)
 }
 
 static DeployResult deploy(const Options &options, const QMap<QString, QString> &qtpathsVariables,
-                           QString *errorMessage)
+                           const PluginInformation &pluginInfo, QString *errorMessage)
 {
     DeployResult result;
 
@@ -1423,8 +1508,8 @@ static DeployResult deploy(const Options &options, const QMap<QString, QString>
             &result.deployedQtLibraries,
             // For non-QML applications, disable QML to prevent it from being pulled in by the
             // qtaccessiblequick plugin.
-            disabled,
+            disabled, pluginInfo,
-            options.disabledPluginTypes, qtpathsVariables.value(QStringLiteral("QT_INSTALL_PLUGINS")),
+            options.pluginSelections, qtpathsVariables.value(QStringLiteral("QT_INSTALL_PLUGINS")),
             libraryLocation, infix, debugMatchMode, options.platform, &platformPlugin,
             options.deployInsightTrackerPlugin);
 
@@ -1569,7 +1654,8 @@ static DeployResult deploy(const Options &options, const QMap<QString, QString>
 }
 
 static bool deployWebProcess(const QMap<QString, QString> &qtpathsVariables, const char *binaryName,
-                             const Options &sourceOptions, QString *errorMessage)
+                             const PluginInformation &pluginInfo, const Options &sourceOptions,
+                             QString *errorMessage)
 {
     // Copy the web process and its dependencies
     const QString webProcess = webProcessBinary(binaryName, sourceOptions.platform);
@@ -1581,11 +1667,12 @@ static bool deployWebProcess(const QMap<QString, QString> &qtpathsVariables, con
     options.binaries.append(options.directory + u'/' + webProcess);
     options.quickImports = false;
     options.translations = false;
-    return deploy(options, qtpathsVariables, errorMessage);
+    return deploy(options, qtpathsVariables, pluginInfo, errorMessage);
 }
 
 static bool deployWebEngineCore(const QMap<QString, QString> &qtpathsVariables,
-                                const Options &options, bool isDebug, QString *errorMessage)
+                                const PluginInformation &pluginInfo, const Options &options,
+                                bool isDebug, QString *errorMessage)
 {
     static const char *installDataFiles[] = { "icudtl.dat",
                                               "qtwebengine_devtools_resources.pak",
@@ -1599,7 +1686,7 @@ static bool deployWebEngineCore(const QMap<QString, QString> &qtpathsVariables,
         webEngineProcessName.append('d');
     if (optVerboseLevel)
         std::wcout << "Deploying: " << webEngineProcessName.constData() << "...\n";
-    if (!deployWebProcess(qtpathsVariables, webEngineProcessName, options, errorMessage))
+    if (!deployWebProcess(qtpathsVariables, webEngineProcessName, pluginInfo, options, errorMessage))
         return false;
     const QString resourcesSubDir = QStringLiteral("/resources");
     const QString resourcesSourceDir = qtpathsVariables.value(QStringLiteral("QT_INSTALL_DATA"))
@@ -1699,6 +1786,10 @@ int main(int argc, char **argv)
     }
     assignKnownModuleIds();
 
+    // Read the Qt plugin types information from the Qt installation directory.
+    PluginInformation pluginInfo{};
+    pluginInfo.generateAvailablePlugins(qtpathsVariables, options.platform);
+
     // Parse the full command line.
     {
         QCommandLineParser parser;
@@ -1707,7 +1798,7 @@ int main(int argc, char **argv)
         if (result & CommandLineParseError)
             std::wcerr << errorMessage << "\n\n";
         if (result & CommandLineParseHelpRequested)
-            std::fputs(qPrintable(helpText(parser)), stdout);
+            std::fputs(qPrintable(helpText(parser, pluginInfo)), stdout);
         if (result & CommandLineParseError)
             return 1;
         if (result & CommandLineParseHelpRequested)
@@ -1725,14 +1816,15 @@ int main(int argc, char **argv)
         return 1;
     }
 
-    const DeployResult result = deploy(options, qtpathsVariables, &errorMessage);
+    const DeployResult result = deploy(options, qtpathsVariables, pluginInfo, &errorMessage);
     if (!result) {
         std::wcerr << errorMessage << '\n';
         return 1;
     }
 
     if (result.deployedQtLibraries.test(QtWebEngineCoreModuleId)) {
-        if (!deployWebEngineCore(qtpathsVariables, options, result.isDebug, &errorMessage)) {
+        if (!deployWebEngineCore(qtpathsVariables, pluginInfo, options, result.isDebug,
+                                 &errorMessage)) {
             std::wcerr << errorMessage << '\n';
             return 1;
         }

src/tools/windeployqt/qtplugininfo.cpp

@@ -0,0 +1,100 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include "qtplugininfo.h"
+
+#include <QDir>
+
+static PluginDetection determinePluginLibrary(const QDir &platformPluginDir, const QString &infix)
+{
+    // Use the platform plugin to determine which dlls are there (release/debug/both)
+    QString platformReleaseFilter(QStringLiteral("qwindows"));
+    if (!infix.isEmpty())
+        platformReleaseFilter += infix;
+    QString platformFilter = platformReleaseFilter + u'*';
+    platformFilter += sharedLibrarySuffix();
+
+    const QFileInfoList &dlls =
+            platformPluginDir.entryInfoList(QStringList(platformFilter), QDir::Files);
+    if (dlls.size() == 1) {
+        const QFileInfo dllFi = dlls.first();
+        const bool hasDebugDlls =
+                dllFi.fileName() == QString(platformReleaseFilter + sharedLibrarySuffix()) ? false
+                                                                                           : true;
+        return (hasDebugDlls ? PluginDetection::DebugOnly : PluginDetection::ReleaseOnly);
+    } else {
+        return PluginDetection::DebugAndRelease;
+    }
+}
+
+static QStringList findPluginNames(const QDir &pluginDir, const PluginDetection libraryType,
+                                   const Platform &platform)
+{
+    QString errorMessage{};
+    QStringList result{};
+    QString filter{};
+    filter += u"*";
+    filter += sharedLibrarySuffix();
+
+    const QFileInfoList &dlls =
+            pluginDir.entryInfoList(QStringList(filter), QDir::Files, QDir::Name);
+
+    for (const QFileInfo &dllFi : dlls) {
+        QString plugin = dllFi.fileName();
+        const int dotIndex = plugin.lastIndexOf(u'.');
+        // We don't need the .dll for the name
+        plugin = plugin.first(dotIndex);
+
+        if (libraryType == PluginDetection::DebugAndRelease) {
+            bool isDebugDll{};
+            if (!readPeExecutable(dllFi.absoluteFilePath(), &errorMessage, 0, 0, &isDebugDll,
+                                  (platform == WindowsDesktopMinGW))) {
+                std::wcerr << "Warning: Unable to read "
+                           << QDir::toNativeSeparators(dllFi.absoluteFilePath()) << ": "
+                           << errorMessage;
+            }
+            if (isDebugDll && platformHasDebugSuffix(platform))
+                plugin.removeLast();
+        }
+        else if (libraryType == PluginDetection::DebugOnly)
+            plugin.removeLast();
+
+        if (!result.contains(plugin))
+            result.append(plugin);
+    }
+    return result;
+}
+
+bool PluginInformation::isTypeForPlugin(const QString &type, const QString &plugin) const
+{
+    return m_pluginMap.at(plugin) == type;
+}
+
+void PluginInformation::populatePluginToType(const QDir &pluginDir, const QStringList &plugins)
+{
+    for (const QString &plugin : plugins)
+        m_pluginMap.insert({ plugin, pluginDir.dirName() });
+}
+
+void PluginInformation::generateAvailablePlugins(const QMap<QString, QString> &qtPathsVariables,
+                                                 const Platform &platform)
+{
+    const QDir pluginTypesDir(qtPathsVariables.value(QLatin1String("QT_INSTALL_PLUGINS")));
+    const QDir platformPluginDir(pluginTypesDir.absolutePath() + QStringLiteral("/platforms"));
+    const QString infix(qtPathsVariables.value(QLatin1String(qmakeInfixKey)));
+    const PluginDetection debugDetection = determinePluginLibrary(platformPluginDir, infix);
+
+    const QFileInfoList &pluginTypes =
+            pluginTypesDir.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot);
+    for (const QFileInfo &pluginType : pluginTypes) {
+        const QString pluginTypeName = pluginType.baseName();
+        m_typeMap.insert({ pluginTypeName, QStringList{} });
+        const QStringList plugins =
+                findPluginNames(pluginType.absoluteFilePath(), debugDetection, platform);
+        m_typeMap.at(pluginTypeName) = plugins;
+        populatePluginToType(pluginTypeName, plugins);
+    }
+    if (!m_typeMap.size() || !m_pluginMap.size())
+        std::wcerr << "Warning: could not parse available plugins properly, plugin "
+                      "inclusion/exclusion options will not work\n";
+}

src/tools/windeployqt/qtplugininfo.h

@@ -0,0 +1,47 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#ifndef QTPLUGININFO_H
+#define QTPLUGININFO_H
+
+#include "utils.h"
+
+#include <QString>
+#include <QStringList>
+
+#include <unordered_map>
+
+enum class PluginDetection
+{
+    DebugOnly,
+    ReleaseOnly,
+    DebugAndRelease
+};
+
+struct PluginLists
+{
+    QStringList disabledPluginTypes;
+    QStringList enabledPluginTypes;
+    QStringList excludedPlugins;
+    QStringList includedPlugins;
+};
+
+class PluginInformation
+{
+public:
+    PluginInformation() = default;
+
+    bool isTypeForPlugin(const QString &type, const QString &plugin) const;
+
+    void generateAvailablePlugins(const QMap<QString, QString> &qtPathsVariables,
+                                  const Platform &platform);
+    void populatePluginToType(const QDir &pluginDir, const QStringList &plugins);
+
+    const std::unordered_map<QString, QStringList> &typeMap() const { return m_typeMap; }
+
+private:
+    std::unordered_map<QString, QStringList> m_typeMap;
+    std::unordered_map<QString, QString> m_pluginMap;
+};
+
+#endif