QProcess/Unix: add a flag to disable core dumps

[ChangeLog][QtCore][QProcess] Added UnixProcessFlag::DisableCoreDumps.

Change-Id: I8c763c4e520fe9347856fffd457b5082696826d3
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@gmx.de>
Reviewed-by: Ahmad Samir <a.samirh78@gmail.com>

src/corelib/io/qprocess.cpp

@@ -896,6 +896,16 @@ void QProcessPrivate::Channel::clear()
            if QProcess will actually use \c{vfork(2)} and if \c{vfork(2)} is
            different from standard \c{fork(2)}.
 
+    \value [since 6.9] DisableCoreDumps     Requests that QProcess disable core
+           dumps in the child process. This is useful if the executable being
+           run is likely to crash but users and maintainers are going to be
+           uninterested in generating bug reports for those conditions (for
+           example, the executable is a test process). This setting does not
+           affect the exitStatus() of the crashed process. It is implemented
+           by setting the core dump size resource soft limit to zero, meaning
+           the application can still reverse this change by raising it to a
+           value up to the hard limit.
+
     \sa setUnixProcessParameters(), unixProcessParameters()
 */
 

src/corelib/io/qprocess.h

@@ -191,6 +191,7 @@ public:
         CreateNewSession                    = 0x0040, // like POSIX_SPAWN_SETSID
         DisconnectControllingTerminal       = 0x0080,
         ResetIds                            = 0x0100, // like POSIX_SPAWN_RESETIDS
+        DisableCoreDumps                    = 0x0200,
     };
     Q_DECLARE_FLAGS(UnixProcessFlags, UnixProcessFlag)
     struct UnixProcessParameters

src/corelib/io/qprocess_unix.cpp

@@ -885,6 +885,15 @@ static const char *applyProcessParameters(const QProcess::UnixProcessParameters
         }
     }
 
+    // Disable core dumps near the end. This isn't expected to fail.
+    if (struct rlimit lim; getrlimit(RLIMIT_CORE, &lim) == 0 && lim.rlim_cur) {
+        // We'll leave rlim_max untouched, so the child can set it back if it
+        // wants to. We don't expect setrlimit() to fail, so we ignore its
+        // return value.
+        lim.rlim_cur = 0;
+        (void) setrlimit(RLIMIT_CORE, &lim);
+    }
+
     // Apply UID and GID parameters last. This isn't expected to fail either:
     // either we're trying to impersonate what we already are, or we're EUID or
     // EGID root, in which case we are allowed to do this.

tests/auto/corelib/io/qprocess/testUnixProcessParameters/main.cpp

@@ -96,6 +96,18 @@ int main(int argc, char **argv)
         return EXIT_FAILURE;
     }
 
+    if (cmd == "no-coredumps") {
+        struct rlimit corelimit;
+        if (getrlimit(RLIMIT_CORE, &corelimit) != 0) {
+            // this shouldn't happen, so just assume it worked
+            return EXIT_SUCCESS;
+        }
+        if (corelimit.rlim_cur == 0)
+            return EXIT_SUCCESS;
+        fprintf(stderr, "rlim_cur = %lx\n", (unsigned long)corelimit.rlim_cur);
+        return EXIT_FAILURE;
+    }
+
     if (cmd == "setsid") {
         pid_t pgid = getpgrp();
         if (pgid == getpid())

tests/auto/corelib/io/qprocess/tst_qprocess.cpp

@@ -1774,6 +1774,7 @@ void tst_QProcess::unixProcessParameters_data()
     addRow("file-descriptors", P::CloseFileDescriptors);
     addRow("setsid", P::CreateNewSession);
     addRow("reset-ids", P::ResetIds);
+    addRow("no-coredumps", P::DisableCoreDumps);
 
     // On FreeBSD, we need to be session leader to disconnect from the CTTY
     addRow("noctty", P::DisconnectControllingTerminal | P::CreateNewSession);
@@ -1788,6 +1789,7 @@ void tst_QProcess::unixProcessParameters()
     struct Scope {
         int devnull;
         struct sigaction old_sigusr1, old_sigpipe;
+        struct rlimit old_corelimit = {};
         Scope()
         {
             int fd = open("/dev/null", O_RDONLY);
@@ -1806,6 +1808,13 @@ void tst_QProcess::unixProcessParameters()
             sigset_t *set = &act.sa_mask;               // reuse this sigset_t
             sigaddset(set, SIGUSR2);
             sigprocmask(SIG_BLOCK, set, nullptr);
+
+            if (getrlimit(RLIMIT_CORE, &old_corelimit) == 0 && old_corelimit.rlim_max) {
+                struct rlimit new_corelimit = old_corelimit;
+                new_corelimit.rlim_cur = new_corelimit.rlim_max;
+                if (setrlimit(RLIMIT_CORE, &new_corelimit) != 0)
+                    old_corelimit = {};
+            }
         }
         ~Scope()
         {
@@ -1822,6 +1831,9 @@ void tst_QProcess::unixProcessParameters()
             sigset_t *set = &old_sigusr1.sa_mask;       // reuse this sigset_t
             sigaddset(set, SIGUSR2);
             sigprocmask(SIG_BLOCK, set, nullptr);
+
+            if (old_corelimit.rlim_max)
+                setrlimit(RLIMIT_CORE, &old_corelimit);
         }
     } scope;
 
@@ -1837,6 +1849,11 @@ void tst_QProcess::unixProcessParameters()
         }
     }
 
+    if (params.flags & QProcess::UnixProcessFlag::DisableCoreDumps
+            && scope.old_corelimit.rlim_max == 0) {
+        QSKIP("Cannot raise the core size limit (hard limit is set to zero)");
+    }
+
     QProcess process;
     process.setUnixProcessParameters(params);
     process.setStandardInputFile(QProcess::nullDevice());   // so we can't mess with SIGPIPE