Simplify the console detection for QT_WIN_DEBUG_CONSOLE

Instead of reading the PE header we can try to retrieve the stdout
handle, and if it's present, we have a console, or redirected output.

This amends commit 639437cf34783573e14d35ceb68ee8d9de495d7b.

Pick-to: 6.7
Task-number: QTBUG-127732
Change-Id: Ib6dd1a37552519fd867e7c39e588a085513f97e0
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@gmx.de>
Reviewed-by: Zhao Yuhang <yuhangzhao@deepin.org>
(cherry picked from commit dfa52e3ae63432c1bfbb70b258ba96f7bfa5d3b6)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
This commit is contained in:
Joerg Bornemann 2024-08-19 14:50:09 +02:00 committed by Qt Cherry-pick Bot
parent a5752f4cb4
commit 21cca5bb93

View File

@ -579,25 +579,17 @@ QString qAppName()
} }
#ifdef Q_OS_WINDOWS #ifdef Q_OS_WINDOWS
// Return true if we could determine that the current process was linked with /SUBSYSTEM:CONSOLE. static bool hasValidStdOutHandle()
// Return false otherwise.
static bool isConsoleApplication()
{ {
auto dosHeader = reinterpret_cast<PIMAGE_DOS_HEADER>(GetModuleHandle(nullptr)); const HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE);
if (!dosHeader || dosHeader->e_magic != IMAGE_DOS_SIGNATURE) return h != NULL && h != INVALID_HANDLE_VALUE;
return false;
auto dosHeaderAddr = reinterpret_cast<PBYTE>(dosHeader);
auto ntHeaders = reinterpret_cast<PIMAGE_NT_HEADERS>(dosHeaderAddr + dosHeader->e_lfanew);
if (ntHeaders->Signature != IMAGE_NT_SIGNATURE)
return false;
return ntHeaders->OptionalHeader.Subsystem == IMAGE_SUBSYSTEM_WINDOWS_CUI;
} }
#endif #endif
void QCoreApplicationPrivate::initConsole() void QCoreApplicationPrivate::initConsole()
{ {
#ifdef Q_OS_WINDOWS #ifdef Q_OS_WINDOWS
if (isConsoleApplication()) if (hasValidStdOutHandle())
return; return;
const QString env = qEnvironmentVariable("QT_WIN_DEBUG_CONSOLE"); const QString env = qEnvironmentVariable("QT_WIN_DEBUG_CONSOLE");
if (env.isEmpty()) if (env.isEmpty())