OS X QColorDialog: emit reject() when closed by the titlebar button

but only if there is a Cancel button.  A color dialog without a cancel
button might be kept open and apply to various selections, so it
doesn't make sense to reject such a dialog, only to close it.

Task-number: QTBUG-40855
Change-Id: Ifffb4ae81307c72259ed388a4776ba09543603e7
Reviewed-by: Gabriel de Dietrich <gabriel.dedietrich@digia.com>

src/plugins/platforms/cocoa/qcocoacolordialoghelper.mm

@@ -81,6 +81,7 @@ static NSButton *macCreateButton(const char *text, NSView *superview)
     NSInteger mResultCode;
     BOOL mDialogIsExecuting;
     BOOL mResultSet;
+    BOOL mClosingDueToKnownButton;
 };
 - (void)restoreOriginalContentView;
 - (void)relayout;
@@ -103,6 +104,7 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QNSColorPanelDelegate);
     mResultCode = NSCancelButton;
     mDialogIsExecuting = false;
     mResultSet = false;
+    mClosingDueToKnownButton = false;
 
 #if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7
     if (QSysInfo::MacintoshVersion >= QSysInfo::MV_10_7)
@@ -114,6 +116,11 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QNSColorPanelDelegate);
         name:NSColorPanelColorDidChangeNotification
         object:mColorPanel];
 
+    [[NSNotificationCenter defaultCenter] addObserver:self
+      selector:@selector(windowWillClose:)
+      name:NSWindowWillCloseNotification
+      object:mColorPanel];
+
     [mColorPanel retain];
     return self;
 }
@@ -179,6 +186,15 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QNSColorPanelDelegate);
         emit mHelper->colorSelected(mQtColor);
 }
 
+- (void)windowWillClose:(NSNotification *)notification
+{
+    Q_UNUSED(notification);
+    if (mCancelButton && mHelper && !mClosingDueToKnownButton) {
+        mClosingDueToKnownButton = true; // prevent repeating emit
+        emit mHelper->reject();
+    }
+}
+
 - (void)restoreOriginalContentView
 {
     if (mStolenContentView) {
@@ -246,6 +262,7 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QNSColorPanelDelegate);
 
 - (void)onOkClicked
 {
+    mClosingDueToKnownButton = true;
     [mColorPanel close];
     [self updateQtColor];
     [self finishOffWithCode:NSOKButton];
@@ -254,6 +271,7 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QNSColorPanelDelegate);
 - (void)onCancelClicked
 {
     if (mOkButton) {
+        mClosingDueToKnownButton = true;
         [mColorPanel close];
         mQtColor = QColor();
         [self finishOffWithCode:NSCancelButton];
@@ -298,6 +316,7 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QNSColorPanelDelegate);
 {
     mDialogIsExecuting = false;
     mResultSet = false;
+    mClosingDueToKnownButton = false;
     [mColorPanel makeKeyAndOrderFront:mColorPanel];
 }