From d6f4a80e570e928739fcdbfd4eeb9b1ade805074 Mon Sep 17 00:00:00 2001
From: "Norvald H. Ryeng" <norvald.ryeng@oracle.com>
Date: Tue, 30 Aug 2011 09:56:07 +0200
Subject: [PATCH] Bug#11765254 - 58200: ASSERTION FAILED: PARAM.SORT_LENGTH
 WHEN GROUPING BY FUNCTIONS.... (PART

The bug was introduced in a patch for bug 49897.

Problem: The assertion inserted by the original patch to guard against
zero-lenght sort keys during merge phase triggers also when the whole
set fits in memory.

Fix: Move assert so that it does not trigger if the whole set is in
memory.
---
 mysql-test/r/group_by.result | 32 ++++++++++++++++++++++++++++++++
 mysql-test/t/group_by.test   | 14 ++++++++++++++
 sql/filesort.cc              |  5 +++--
 3 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result
index 1dfb0f5860a..2c9f6116eed 100644
--- a/mysql-test/r/group_by.result
+++ b/mysql-test/r/group_by.result
@@ -1891,4 +1891,36 @@ a	AVG(t1.b)	t11c	t12c
 1	4.0000	6	6
 2	2.0000	7	7
 DROP TABLE t1;
+#
+# Bug#11765254 (58200): Assertion failed: param.sort_length when grouping
+# by functions
+#
+SET SQL_BIG_TABLES=1;
+CREATE TABLE t1(a INT);
+INSERT INTO t1 VALUES (0),(0);
+SELECT 1 FROM t1 GROUP BY IF(`a`,'','');
+1
+1
+SELECT 1 FROM t1 GROUP BY TRIM(LEADING RAND() FROM '');
+1
+1
+SELECT 1 FROM t1 GROUP BY SUBSTRING('',SLEEP(0),'');
+1
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: ''
+Warning	1292	Truncated incorrect INTEGER value: ''
+Warning	1292	Truncated incorrect INTEGER value: ''
+SELECT 1 FROM t1 GROUP BY SUBSTRING(SYSDATE() FROM 'K' FOR 'jxW<');
+1
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+DROP TABLE t1;
+SET SQL_BIG_TABLES=0;
 # End of 5.1 tests
diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test
index 1a4b9a3bab7..29ed26215bf 100644
--- a/mysql-test/t/group_by.test
+++ b/mysql-test/t/group_by.test
@@ -1283,5 +1283,19 @@ FROM t1 GROUP BY a;
 
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#11765254 (58200): Assertion failed: param.sort_length when grouping
+--echo # by functions
+--echo #
+
+SET SQL_BIG_TABLES=1;
+CREATE TABLE t1(a INT);
+INSERT INTO t1 VALUES (0),(0);
+SELECT 1 FROM t1 GROUP BY IF(`a`,'','');
+SELECT 1 FROM t1 GROUP BY TRIM(LEADING RAND() FROM '');
+SELECT 1 FROM t1 GROUP BY SUBSTRING('',SLEEP(0),'');
+SELECT 1 FROM t1 GROUP BY SUBSTRING(SYSDATE() FROM 'K' FOR 'jxW<');
+DROP TABLE t1;
+SET SQL_BIG_TABLES=0;
 
 --echo # End of 5.1 tests
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 0ff354b334c..715528bcd52 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -144,8 +144,6 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   error= 1;
   bzero((char*) &param,sizeof(param));
   param.sort_length= sortlength(thd, sortorder, s_length, &multi_byte_charset);
-  /* filesort cannot handle zero-length records. */
-  DBUG_ASSERT(param.sort_length);
   param.ref_length= table->file->ref_length;
   param.addon_field= 0;
   param.addon_length= 0;
@@ -257,6 +255,9 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   }
   else
   {
+    /* filesort cannot handle zero-length records during merge. */
+    DBUG_ASSERT(param.sort_length != 0);
+
     if (table_sort.buffpek && table_sort.buffpek_len < maxbuffer)
     {
       x_free(table_sort.buffpek);