Win: redirect console output to the parent process if needed

We need to redirect the console output to the parent process
for GUI applications on Windows, otherwise we won't get any
output if we start the application from the console, because
the SUBSYSTEM of GUI applications is not CONSOLE by default.

But we don't want to change the default behavior of Qt, so
we control this feature through an environment variable
"QT_WIN_DEBUG_CONSOLE". It accepts two string values:
(1) "new": the application will try to create a separate
    console window and redirect everything (cin/cout/clog/cerr)
    to it. If you are running the application in an IDE, you
    won't be able to get anything from the IDE's console anymore.
(2) "attach": the application will try to attach to the parent
    process's console, if there is one. When we attached to it
    successfully, we'll redirect everything to it.

Change-Id: I3ef98f6c0603f64fcc4e8e974411c5ed83c5d36f
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@gmx.de>
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

src/corelib/kernel/qcoreapplication.cpp

@@ -451,6 +451,8 @@ QCoreApplicationPrivate::~QCoreApplicationPrivate()
 #endif
 #if defined(Q_OS_WIN)
     delete [] origArgv;
+    if (consoleAllocated)
+        FreeConsole();
 #endif
     QCoreApplicationPrivate::clearApplicationFilePath();
 }
@@ -549,6 +551,37 @@ QString qAppName()
     return QCoreApplication::instance()->d_func()->appName();
 }
 
+void QCoreApplicationPrivate::initConsole()
+{
+#ifdef Q_OS_WINDOWS
+    const QString env = qEnvironmentVariable("QT_WIN_DEBUG_CONSOLE");
+    if (env.isEmpty())
+        return;
+    if (env.compare(u"new"_s, Qt::CaseInsensitive) == 0) {
+        if (AllocConsole() == FALSE)
+            return;
+        consoleAllocated = true;
+    } else if (env.compare(u"attach"_s, Qt::CaseInsensitive) == 0) {
+        if (AttachConsole(ATTACH_PARENT_PROCESS) == FALSE)
+            return;
+    } else {
+        // Unknown input, don't make any decision for the user.
+        return;
+    }
+    // The std{in,out,err} handles are read-only, so we need to pass in dummies.
+    FILE *in = nullptr;
+    FILE *out = nullptr;
+    FILE *err = nullptr;
+    freopen_s(&in, "CONIN$", "r", stdin);
+    freopen_s(&out, "CONOUT$", "w", stdout);
+    freopen_s(&err, "CONOUT$", "w", stderr);
+    // However, things wouldn't work if the runtime did not preserve the pointers.
+    Q_ASSERT(in == stdin);
+    Q_ASSERT(out == stdout);
+    Q_ASSERT(err == stderr);
+#endif
+}
+
 void QCoreApplicationPrivate::initLocale()
 {
 #if defined(Q_OS_UNIX) && !defined(QT_BOOTSTRAPPED)
@@ -744,6 +777,8 @@ void QCoreApplicationPrivate::init()
 
     Q_Q(QCoreApplication);
 
+    initConsole();
+
     initLocale();
 
     Q_ASSERT_X(!QCoreApplication::self, "QCoreApplication", "there should be only one application object");

src/corelib/kernel/qcoreapplication_p.h

@@ -73,6 +73,7 @@ public:
     static QString infoDictionaryStringProperty(const QString &propertyName);
 #endif
 
+    void initConsole();
     static void initLocale();
 
     static bool checkInstance(const char *method);
@@ -125,6 +126,7 @@ public:
 #if defined(Q_OS_WIN)
     int origArgc;
     char **origArgv; // store unmodified arguments for QCoreApplication::arguments()
+    bool consoleAllocated = false;
 #endif
     void appendApplicationPathToLibraryPaths(void);