From 21cca5bb932a90be6c64146787dabd54146e651c Mon Sep 17 00:00:00 2001 From: Joerg Bornemann Date: Mon, 19 Aug 2024 14:50:09 +0200 Subject: [PATCH] Simplify the console detection for QT_WIN_DEBUG_CONSOLE Instead of reading the PE header we can try to retrieve the stdout handle, and if it's present, we have a console, or redirected output. This amends commit 639437cf34783573e14d35ceb68ee8d9de495d7b. Pick-to: 6.7 Task-number: QTBUG-127732 Change-Id: Ib6dd1a37552519fd867e7c39e588a085513f97e0 Reviewed-by: Thiago Macieira Reviewed-by: Oswald Buddenhagen Reviewed-by: Zhao Yuhang (cherry picked from commit dfa52e3ae63432c1bfbb70b258ba96f7bfa5d3b6) Reviewed-by: Qt Cherry-pick Bot --- src/corelib/kernel/qcoreapplication.cpp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/corelib/kernel/qcoreapplication.cpp b/src/corelib/kernel/qcoreapplication.cpp index cde04bd81ea..dd9b6acf4a3 100644 --- a/src/corelib/kernel/qcoreapplication.cpp +++ b/src/corelib/kernel/qcoreapplication.cpp @@ -579,25 +579,17 @@ QString qAppName() } #ifdef Q_OS_WINDOWS -// Return true if we could determine that the current process was linked with /SUBSYSTEM:CONSOLE. -// Return false otherwise. -static bool isConsoleApplication() +static bool hasValidStdOutHandle() { - auto dosHeader = reinterpret_cast(GetModuleHandle(nullptr)); - if (!dosHeader || dosHeader->e_magic != IMAGE_DOS_SIGNATURE) - return false; - auto dosHeaderAddr = reinterpret_cast(dosHeader); - auto ntHeaders = reinterpret_cast(dosHeaderAddr + dosHeader->e_lfanew); - if (ntHeaders->Signature != IMAGE_NT_SIGNATURE) - return false; - return ntHeaders->OptionalHeader.Subsystem == IMAGE_SUBSYSTEM_WINDOWS_CUI; + const HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE); + return h != NULL && h != INVALID_HANDLE_VALUE; } #endif void QCoreApplicationPrivate::initConsole() { #ifdef Q_OS_WINDOWS - if (isConsoleApplication()) + if (hasValidStdOutHandle()) return; const QString env = qEnvironmentVariable("QT_WIN_DEBUG_CONSOLE"); if (env.isEmpty())