From cef270265fd9bd299e2fe5bb45a7c14002cfe1bf Mon Sep 17 00:00:00 2001
From: Joerg Bornemann <joerg.bornemann@qt.io>
Date: Tue, 6 Aug 2024 17:42:56 +0200
Subject: [PATCH] Ignore QT_WIN_DEBUG_CONSOLE for console applications

On Windows, the windeployqt tool failed if QT_WIN_DEBUG_CONSOLE was set
to "attached", because the following happened:
- windeployqt calls qtpaths
- qtpaths attaches to windeployqt's console
- qtpaths' output goes to that console
- windeployqt cannot read qtpaths' output anymore

Other Qt-based command line tools are also affected.

The QT_WIN_DEBUG_CONSOLE environment variable was introduced to see
output from Gui applications on the console. It should not affect
console applications.

We now determine whether the current process was linked with
/SUBSYSTEM:CONSOLE and ignore QT_WIN_DEBUG_CONSOLE's value in that case.

Fixes: QTBUG-127732
Pick-to: 6.7 6.8
Change-Id: Iba7031eed88c7b38cfe3e794c1885b504e4f2ee4
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
Reviewed-by: Zhao Yuhang <yuhangzhao@deepin.org>
---
 src/corelib/kernel/qcoreapplication.cpp | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/corelib/kernel/qcoreapplication.cpp b/src/corelib/kernel/qcoreapplication.cpp
index fe1827ef578..4d37894f353 100644
--- a/src/corelib/kernel/qcoreapplication.cpp
+++ b/src/corelib/kernel/qcoreapplication.cpp
@@ -576,9 +576,27 @@ QString qAppName()
     return QCoreApplication::instance()->d_func()->appName();
 }
 
+#ifdef Q_OS_WINDOWS
+// Return true if we could determine that the current process was linked with /SUBSYSTEM:CONSOLE.
+// Return false otherwise.
+static bool isConsoleApplication()
+{
+    auto dosHeader = reinterpret_cast<PIMAGE_DOS_HEADER>(GetModuleHandle(nullptr));
+    if (!dosHeader || dosHeader->e_magic != IMAGE_DOS_SIGNATURE)
+        return false;
+    auto dosHeaderAddr = reinterpret_cast<PBYTE>(dosHeader);
+    auto ntHeaders = reinterpret_cast<PIMAGE_NT_HEADERS>(dosHeaderAddr + dosHeader->e_lfanew);
+    if (ntHeaders->Signature != IMAGE_NT_SIGNATURE)
+        return false;
+    return ntHeaders->OptionalHeader.Subsystem == IMAGE_SUBSYSTEM_WINDOWS_CUI;
+}
+#endif
+
 void QCoreApplicationPrivate::initConsole()
 {
 #ifdef Q_OS_WINDOWS
+    if (isConsoleApplication())
+        return;
     const QString env = qEnvironmentVariable("QT_WIN_DEBUG_CONSOLE");
     if (env.isEmpty())
         return;