Simplify the console detection for QT_WIN_DEBUG_CONSOLE

Instead of reading the PE header we can try to retrieve the stdout
handle, and if it's present, we have a console, or redirected output.

This amends commit 639437cf34783573e14d35ceb68ee8d9de495d7b.

Pick-to: 6.7
Task-number: QTBUG-127732
Change-Id: Ib6dd1a37552519fd867e7c39e588a085513f97e0
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@gmx.de>
Reviewed-by: Zhao Yuhang <yuhangzhao@deepin.org>
(cherry picked from commit dfa52e3ae63432c1bfbb70b258ba96f7bfa5d3b6)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
This commit is contained in:
Joerg Bornemann 2024-08-19 14:50:09 +02:00 committed by Qt Cherry-pick Bot
parent a5752f4cb4
commit 21cca5bb93

View File

@ -579,25 +579,17 @@ QString qAppName()
}
#ifdef Q_OS_WINDOWS
// Return true if we could determine that the current process was linked with /SUBSYSTEM:CONSOLE.
// Return false otherwise.
static bool isConsoleApplication()
static bool hasValidStdOutHandle()
{
auto dosHeader = reinterpret_cast<PIMAGE_DOS_HEADER>(GetModuleHandle(nullptr));
if (!dosHeader || dosHeader->e_magic != IMAGE_DOS_SIGNATURE)
return false;
auto dosHeaderAddr = reinterpret_cast<PBYTE>(dosHeader);
auto ntHeaders = reinterpret_cast<PIMAGE_NT_HEADERS>(dosHeaderAddr + dosHeader->e_lfanew);
if (ntHeaders->Signature != IMAGE_NT_SIGNATURE)
return false;
return ntHeaders->OptionalHeader.Subsystem == IMAGE_SUBSYSTEM_WINDOWS_CUI;
const HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE);
return h != NULL && h != INVALID_HANDLE_VALUE;
}
#endif
void QCoreApplicationPrivate::initConsole()
{
#ifdef Q_OS_WINDOWS
if (isConsoleApplication())
if (hasValidStdOutHandle())
return;
const QString env = qEnvironmentVariable("QT_WIN_DEBUG_CONSOLE");
if (env.isEmpty())