diff --git a/mysql-test/r/order_by_zerolength-4285.result b/mysql-test/r/order_by_zerolength-4285.result index f60ce7d90c7..e4c117b26af 100644 --- a/mysql-test/r/order_by_zerolength-4285.result +++ b/mysql-test/r/order_by_zerolength-4285.result @@ -24,3 +24,23 @@ Warning 1292 Truncated incorrect CHAR(0) value: '8' Warning 1292 Truncated incorrect CHAR(0) value: '9' Warning 1292 Truncated incorrect CHAR(0) value: '10' drop table t1; +# +# MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion +# +set @save_sql_mode= @@sql_mode; +SET @@sql_mode= ''; +CREATE TABLE t1 (pk INT PRIMARY KEY); +INSERT INTO t1 VALUES (1),(2); +explain +SELECT * FROM t1 ORDER BY 'foo', CONVERT(pk, CHAR(0)) LIMIT 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 4 NULL 2 Using index; Using filesort +SELECT * FROM t1 ORDER BY 'foo', Cast(pk as CHAR(0)) LIMIT 2; +pk +1 +2 +Warnings: +Warning 1292 Truncated incorrect CHAR(0) value: '1' +Warning 1292 Truncated incorrect CHAR(0) value: '2' +set @@sql_mode= @save_sql_mode; +drop table t1; diff --git a/mysql-test/t/order_by_zerolength-4285.test b/mysql-test/t/order_by_zerolength-4285.test index 2fb58edd36d..9533f2cc62e 100644 --- a/mysql-test/t/order_by_zerolength-4285.test +++ b/mysql-test/t/order_by_zerolength-4285.test @@ -6,3 +6,16 @@ insert into t1 values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); select * from t1 order by now(), cast(pk as char(0)); drop table t1; +--echo # +--echo # MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion +--echo # + +set @save_sql_mode= @@sql_mode; +SET @@sql_mode= ''; +CREATE TABLE t1 (pk INT PRIMARY KEY); +INSERT INTO t1 VALUES (1),(2); +explain +SELECT * FROM t1 ORDER BY 'foo', CONVERT(pk, CHAR(0)) LIMIT 2; +SELECT * FROM t1 ORDER BY 'foo', Cast(pk as CHAR(0)) LIMIT 2; +set @@sql_mode= @save_sql_mode; +drop table t1; diff --git a/mysys/ptr_cmp.c b/mysys/ptr_cmp.c index 6e373e98972..c9306dff14f 100644 --- a/mysys/ptr_cmp.c +++ b/mysys/ptr_cmp.c @@ -52,6 +52,11 @@ static int ptr_compare_0(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_1(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_2(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_3(size_t *compare_length, uchar **a, uchar **b); +static int degenerate_compare_func(size_t *compare_length, uchar **a, uchar **b) +{ + DBUG_ASSERT(*compare_length == 0); + return 0; +} #endif /* __sun */ /* Get a pointer to a optimal byte-compare function for a given size */ @@ -64,6 +69,8 @@ qsort2_cmp get_ptr_compare (size_t size __attribute__((unused))) #else qsort2_cmp get_ptr_compare (size_t size) { + if (size == 0) + return (qsort2_cmp) degenerate_compare_func; if (size < 4) return (qsort2_cmp) ptr_compare; switch (size & 3) {