From e9e6679381bf1b3c114302a338b66beee59e9bef Mon Sep 17 00:00:00 2001
From: "evgen@moonbone.local" <>
Date: Fri, 25 Apr 2008 00:39:37 +0400
Subject: [PATCH] Bug#36023: Incorrect handling of zero length caused an
 assertion to fail.

When a zero length is provided to the my_decimal_length_to_precision
function along with unsigned_flag set to false it returns a negative value.
For queries that employs temporary tables may cause failed assertion or
excessive memory consumption while temporary table creation.

Now the my_decimal_length_to_precision and the my_decimal_precision_to_length
functions take unsigned_flag into account only if the length/precision
argument is non-zero.
---
 mysql-test/r/type_decimal.result |  7 +++++++
 mysql-test/t/type_decimal.test   |  8 ++++++++
 sql/my_decimal.h                 | 13 +++++++++++--
 3 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/mysql-test/r/type_decimal.result b/mysql-test/r/type_decimal.result
index 2afd42f702e..03fbc898cc5 100644
--- a/mysql-test/r/type_decimal.result
+++ b/mysql-test/r/type_decimal.result
@@ -946,4 +946,11 @@ SELECT ROUND(20061108085411.000002);
 ROUND(20061108085411.000002)
 20061108085411
 DROP TABLE t1, t2, t3, t4, t5, t6;
+create table t1(`c` decimal(9,2));
+insert into t1 values (300),(201.11);
+select max(case 1 when 1 then c else null end) from t1 group by c;
+max(case 1 when 1 then c else null end)
+201.11
+300.00
+drop table t1;
 End of 5.0 tests
diff --git a/mysql-test/t/type_decimal.test b/mysql-test/t/type_decimal.test
index 6841b3cdd68..8a81908296f 100644
--- a/mysql-test/t/type_decimal.test
+++ b/mysql-test/t/type_decimal.test
@@ -521,4 +521,12 @@ SELECT ROUND(20061108085411.000002);
 
 DROP TABLE t1, t2, t3, t4, t5, t6;
 
+#
+# Bug#36023: Incorrect handling of zero length caused an assertion to fail.
+#
+create table t1(`c` decimal(9,2));
+insert into t1 values (300),(201.11);
+select max(case 1 when 1 then c else null end) from t1 group by c;
+drop table t1;
+
 --echo End of 5.0 tests
diff --git a/sql/my_decimal.h b/sql/my_decimal.h
index c661579ea66..6a0d05921ec 100644
--- a/sql/my_decimal.h
+++ b/sql/my_decimal.h
@@ -164,14 +164,23 @@ inline int check_result_and_overflow(uint mask, int result, my_decimal *val)
 inline uint my_decimal_length_to_precision(uint length, uint scale,
                                            bool unsigned_flag)
 {
-  return (uint) (length - (scale>0 ? 1:0) - (unsigned_flag ? 0:1));
+  /* Precision can't be negative thus ignore unsigned_flag when length is 0. */
+  DBUG_ASSERT(length || !scale);
+  return (uint) (length - (scale>0 ? 1:0) -
+                 (unsigned_flag || !length ? 0:1));
 }
 
 inline uint32 my_decimal_precision_to_length(uint precision, uint8 scale,
                                              bool unsigned_flag)
 {
+  /*
+    When precision is 0 it means that original length was also 0. Thus
+    unsigned_flag is ignored in this case.
+  */
+  DBUG_ASSERT(precision || !scale);
   set_if_smaller(precision, DECIMAL_MAX_PRECISION);
-  return (uint32)(precision + (scale>0 ? 1:0) + (unsigned_flag ? 0:1));
+  return (uint32)(precision + (scale>0 ? 1:0) +
+                  (unsigned_flag || !precision ? 0:1));
 }
 
 inline