iOS: Update RLIMIT_STACK when running with user-main stack

When running the user's main() we retarget the stack pointer to a custom
stack, with half the size of the main thread stack (i.e 512K instead of
1MB). During execution of the user's main() we want code that reads
RLIMIT_STACK to be aware of the limited stack space, as this might impact
where memory guards are placed, or how stack overflow is detected, such
as in the V4 QML engine.

Once we return from the user's main() we restore back to the original
stack size limit. We also handle re-entry into app.exec() as well as
app termination, by always making sure RLIMIT_STACK reflects the available
stack space.

Unfortunately we can't do the same for pthread_get_stacksize_np, as once
the main thread has been created we can't tweak the stack size, and the
pthread_get_stacksize_np function does not query the rlimit on iOS, as
it does in some cases on macOS.

Change-Id: I48e92e762e125cff0f7da2003c7a1e796835d391
Reviewed-by: Ulf Hermann <ulf.hermann@qt.io>
Reviewed-by: Richard Moe Gustavsen <richard.gustavsen@qt.io>

src/plugins/platforms/ios/qioseventdispatcher.mm

@@ -11,6 +11,7 @@
 
 #include <QtCore/qprocessordetection.h>
 #include <QtCore/private/qcoreapplication_p.h>
+#include <QtCore/private/qsystemerror_p.h>
 #include <QtCore/private/qthread_p.h>
 
 #include <qpa/qwindowsysteminterface.h>
@@ -33,6 +34,8 @@
 static const size_t kBytesPerKiloByte = 1024;
 static const long kPageSize = sysconf(_SC_PAGESIZE);
 
+using namespace QT_PREPEND_NAMESPACE(QtPrivate);
+
 /*
     The following diagram shows the layout of the reserved
     stack in relation to the regular stack, and the basic
@@ -79,6 +82,9 @@ static const long kPageSize = sysconf(_SC_PAGESIZE);
 
 namespace
 {
+    rlimit stackLimit = {0, 0};
+    rlim_t originalStackSize = 0;
+
     struct Stack
     {
         uintptr_t base;
@@ -102,8 +108,7 @@ namespace
             stackSize = qMin(stackSize, ((1024 - 64) * kBytesPerKiloByte));
 
             // Which we verify, just in case
-            struct rlimit stackLimit = {0, 0};
-            if (Q_UNLIKELY(getrlimit(RLIMIT_STACK, &stackLimit) == 0 && stackSize > stackLimit.rlim_cur))
+            if (Q_UNLIKELY(stackSize > originalStackSize))
                 qFatal("Unexpectedly exceeded stack limit");
 
             return stackSize;
@@ -179,15 +184,42 @@ namespace
     } logActivity;
 
     static bool s_isQtApplication = false;
-}
 
-using namespace QT_PREPEND_NAMESPACE(QtPrivate);
+    void updateStackLimit()
+    {
+        qCDebug(lcEventDispatcher) << "Updating RLIMIT_STACK soft limit from"
+            << originalStackSize << "to" << userMainStack.size();
+
+        stackLimit.rlim_cur = userMainStack.size();
+        if (setrlimit(RLIMIT_STACK, &stackLimit) != 0) {
+            qCWarning(lcEventDispatcher) << "Failed to update RLIMIT_STACK soft limit"
+                                         << QSystemError::stdString();
+        }
+    }
+
+    void restoreStackLimit()
+    {
+        qCDebug(lcEventDispatcher) << "Restoring RLIMIT_STACK soft limit from"
+            << stackLimit.rlim_cur << "back to" << originalStackSize;
+
+        stackLimit.rlim_cur = originalStackSize;
+        if (setrlimit(RLIMIT_STACK, &stackLimit) != 0) {
+            qCWarning(lcEventDispatcher) << "Failed to update RLIMIT_STACK soft limit"
+                                         << QSystemError::stdString();
+        }
+    }
+}
 
 extern "C" int qt_main_wrapper(int argc, char *argv[])
 {
     s_isQtApplication = true;
 
     @autoreleasepool {
+        if (Q_UNLIKELY(getrlimit(RLIMIT_STACK, &stackLimit) != 0))
+            qFatal("Failed to get stack limits");
+
+        originalStackSize = stackLimit.rlim_cur;
+
         size_t defaultStackSize = 512 * kBytesPerKiloByte; // Same as secondary threads
 
         uint requestedStackSize = qMax(0, infoPlistValue(@"QtRunLoopIntegrationStackSize", defaultStackSize));
@@ -257,6 +289,8 @@ static void __attribute__((noinline, noreturn)) user_main_trampoline()
             qFatal("Could not convert argv[%d] to C string", i);
     }
 
+    updateStackLimit();
+
     int exitCode = main(argc, argv);
     delete[] argv;
 
@@ -266,6 +300,8 @@ static void __attribute__((noinline, noreturn)) user_main_trampoline()
     if (Q_UNLIKELY(debugStackUsage))
         userMainStack.printUsage();
 
+    restoreStackLimit();
+
     logActivity.applicationDidFinishLaunching.leave();
 
     if (applicationAboutToTerminate)
@@ -371,6 +407,8 @@ static bool rootLevelRunLoopIntegration()
         if (Q_UNLIKELY(debugStackUsage))
             userMainStack.printUsage();
 
+        restoreStackLimit();
+
         break;
     default:
         qFatal("Unexpected jump result in event loop integration");
@@ -510,6 +548,7 @@ bool __attribute__((returns_twice)) QIOSJumpingEventDispatcher::processEvents(QE
             // which will emit aboutToQuit if it's QApplication's event loop, and then return to the user's
             // main, which can do whatever it wants, including calling exec() on the application again.
             qCDebug(lcEventDispatcher) << "⇢ System runloop exited, returning with eventsProcessed = true";
+            updateStackLimit();
             return true;
         default:
             qFatal("Unexpected jump result in event loop integration");
@@ -555,6 +594,7 @@ void QIOSJumpingEventDispatcher::interruptEventLoopExec()
         // QEventLoop was re-executed
         logActivity.UIApplicationMain.enter();
         qCDebug(lcEventDispatcher) << "↳ Jumped from processEvents due to re-exec";
+        restoreStackLimit();
         break;
     default:
         qFatal("Unexpected jump result in event loop integration");