1berry/MariaDB-server
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bug#36023: Incorrect handling of zero length caused an assertion to fail.

Browse Source 
When a zero length is provided to the my_decimal_length_to_precision
function along with unsigned_flag set to false it returns a negative value.
For queries that employs temporary tables may cause failed assertion or
excessive memory consumption while temporary table creation.

Now the my_decimal_length_to_precision and the my_decimal_precision_to_length
functions take unsigned_flag into account only if the length/precision
argument is non-zero.
This commit is contained in:
evgen@moonbone.local 2008-04-25 00:39:37 +04:00
parent 9c70737410
commit e9e6679381
3 changed files with 26 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
mysql-test/r/type_decimal.result
View File

@ -946,4 +946,11 @@ SELECT ROUND(20061108085411.000002);
ROUND(20061108085411.000002) ROUND(20061108085411.000002)
20061108085411 20061108085411
DROP TABLE t1, t2, t3, t4, t5, t6; DROP TABLE t1, t2, t3, t4, t5, t6;
create table t1(`c` decimal(9,2));
insert into t1 values (300),(201.11);
select max(case 1 when 1 then c else null end) from t1 group by c;
max(case 1 when 1 then c else null end)
201.11
300.00
drop table t1;
End of 5.0 tests End of 5.0 tests

8
mysql-test/t/type_decimal.test
View File

@ -521,4 +521,12 @@ SELECT ROUND(20061108085411.000002);
DROP TABLE t1, t2, t3, t4, t5, t6; DROP TABLE t1, t2, t3, t4, t5, t6;
#
# Bug#36023: Incorrect handling of zero length caused an assertion to fail.
#
create table t1(`c` decimal(9,2));
insert into t1 values (300),(201.11);
select max(case 1 when 1 then c else null end) from t1 group by c;
drop table t1;
--echo End of 5.0 tests --echo End of 5.0 tests

13
sql/my_decimal.h
View File

@ -164,14 +164,23 @@ inline int check_result_and_overflow(uint mask, int result, my_decimal *val)
inline uint my_decimal_length_to_precision(uint length, uint scale, inline uint my_decimal_length_to_precision(uint length, uint scale,
bool unsigned_flag) bool unsigned_flag)
{ {
return (uint) (length - (scale>0 ? 1:0) - (unsigned_flag ? 0:1)); /* Precision can't be negative thus ignore unsigned_flag when length is 0. */
DBUG_ASSERT(length || !scale);
return (uint) (length - (scale>0 ? 1:0) -
(unsigned_flag || !length ? 0:1));
} }
inline uint32 my_decimal_precision_to_length(uint precision, uint8 scale, inline uint32 my_decimal_precision_to_length(uint precision, uint8 scale,
bool unsigned_flag) bool unsigned_flag)
{ {
/*
When precision is 0 it means that original length was also 0. Thus
unsigned_flag is ignored in this case.
*/
DBUG_ASSERT(precision || !scale);
set_if_smaller(precision, DECIMAL_MAX_PRECISION); set_if_smaller(precision, DECIMAL_MAX_PRECISION);
return (uint32)(precision + (scale>0 ? 1:0) + (unsigned_flag ? 0:1)); return (uint32)(precision + (scale>0 ? 1:0) +
(unsigned_flag || !precision ? 0:1));
} }
inline inline