From ae0cbfe934eacb6744b28fbf9997002ed82c0c49 Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Mon, 11 Nov 2024 12:50:56 +0400
Subject: [PATCH] MDEV-28001 greatest/least with bigint unsigned maxium has
 unexpected results compared to 0

LEAST() and GREATEST() erroneously calcucalted the result as signed
for BIGINT UNSIGNED arguments.

Adding a new method for unsigned arguments:
  Item_func_min_max::val_uint_native()
---
 mysql-test/main/func_hybrid_type.result |  9 +++++++++
 mysql-test/main/func_hybrid_type.test   |  8 ++++++++
 sql/item_func.cc                        | 21 +++++++++++++++++++++
 sql/item_func.h                         |  1 +
 sql/sql_type.cc                         |  2 +-
 5 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/mysql-test/main/func_hybrid_type.result b/mysql-test/main/func_hybrid_type.result
index f20aad9d726..011f97c80f8 100644
--- a/mysql-test/main/func_hybrid_type.result
+++ b/mysql-test/main/func_hybrid_type.result
@@ -4338,5 +4338,14 @@ SELECT * FROM t0 WHERE LEAST(c0, NULL);
 c0
 DROP TABLE t0;
 #
+#   MDEV-28001 greatest/least with bigint unsigned maxium has unexpected results compared to 0
+#
+CREATE TABLE t1 (a BIGINT UNSIGNED, b BIGINT UNSIGNED);
+INSERT INTO t1 VALUES (18446744073709551615,0);
+SELECT a,b, LEAST(a,b), GREATEST(a,b) FROM t1;
+a	b	LEAST(a,b)	GREATEST(a,b)
+18446744073709551615	0	0	18446744073709551615
+DROP TABLE t1;
+#
 # End of 10.5 tests
 #
diff --git a/mysql-test/main/func_hybrid_type.test b/mysql-test/main/func_hybrid_type.test
index 2ebfb3cfdf1..a5ed5d9d604 100644
--- a/mysql-test/main/func_hybrid_type.test
+++ b/mysql-test/main/func_hybrid_type.test
@@ -1137,6 +1137,14 @@ SELECT * FROM t0 WHERE GREATEST(c0, NULL);
 SELECT * FROM t0 WHERE LEAST(c0, NULL);
 DROP TABLE t0;
 
+--echo #
+--echo #   MDEV-28001 greatest/least with bigint unsigned maxium has unexpected results compared to 0
+--echo #
+
+CREATE TABLE t1 (a BIGINT UNSIGNED, b BIGINT UNSIGNED);
+INSERT INTO t1 VALUES (18446744073709551615,0);
+SELECT a,b, LEAST(a,b), GREATEST(a,b) FROM t1;
+DROP TABLE t1;
 
 --echo #
 --echo # End of 10.5 tests
diff --git a/sql/item_func.cc b/sql/item_func.cc
index 553a044eb83..f25ca5ed408 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -3051,6 +3051,27 @@ longlong Item_func_min_max::val_int_native()
 }
 
 
+longlong Item_func_min_max::val_uint_native()
+{
+  DBUG_ASSERT(fixed);
+  ulonglong value= 0;
+  for (uint i=0; i < arg_count ; i++)
+  {
+    if (i == 0)
+      value= (ulonglong) args[i]->val_int();
+    else
+    {
+      ulonglong tmp= (ulonglong) args[i]->val_int();
+      if (!args[i]->null_value && (tmp < value ? cmp_sign : -cmp_sign) > 0)
+	value= tmp;
+    }
+    if ((null_value= args[i]->null_value))
+      return 0;
+  }
+  return (longlong) value;
+}
+
+
 my_decimal *Item_func_min_max::val_decimal_native(my_decimal *dec)
 {
   DBUG_ASSERT(fixed == 1);
diff --git a/sql/item_func.h b/sql/item_func.h
index 1ac6045ee61..f4f0558a77b 100644
--- a/sql/item_func.h
+++ b/sql/item_func.h
@@ -2118,6 +2118,7 @@ public:
   String *val_str_native(String *str);
   double val_real_native();
   longlong val_int_native();
+  longlong val_uint_native();
   my_decimal *val_decimal_native(my_decimal *);
   bool get_date_native(THD *thd, MYSQL_TIME *res, date_mode_t fuzzydate);
   bool get_time_native(THD *thd, MYSQL_TIME *res);
diff --git a/sql/sql_type.cc b/sql/sql_type.cc
index 98abf154db4..7727610088e 100644
--- a/sql/sql_type.cc
+++ b/sql/sql_type.cc
@@ -6152,7 +6152,7 @@ longlong Type_handler_timestamp_common::
 longlong Type_handler_numeric::
          Item_func_min_max_val_int(Item_func_min_max *func) const
 {
-  return func->val_int_native();
+  return is_unsigned() ? func->val_uint_native() : func->val_int_native();
 }