diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result
index 2a52d63e1d7..950de54b815 100644
--- a/mysql-test/r/group_by.result
+++ b/mysql-test/r/group_by.result
@@ -1892,6 +1892,38 @@ a	AVG(t1.b)	t11c	t12c
 1	4.0000	6	6
 2	2.0000	7	7
 DROP TABLE t1;
+#
+# Bug#11765254 (58200): Assertion failed: param.sort_length when grouping
+# by functions
+#
+SET SQL_BIG_TABLES=1;
+CREATE TABLE t1(a INT);
+INSERT INTO t1 VALUES (0),(0);
+SELECT 1 FROM t1 GROUP BY IF(`a`,'','');
+1
+1
+SELECT 1 FROM t1 GROUP BY TRIM(LEADING RAND() FROM '');
+1
+1
+SELECT 1 FROM t1 GROUP BY SUBSTRING('',SLEEP(0),'');
+1
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: ''
+Warning	1292	Truncated incorrect INTEGER value: ''
+Warning	1292	Truncated incorrect INTEGER value: ''
+SELECT 1 FROM t1 GROUP BY SUBSTRING(SYSDATE() FROM 'K' FOR 'jxW<');
+1
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+DROP TABLE t1;
+SET SQL_BIG_TABLES=0;
 # End of 5.1 tests
 #
 # Bug#49771: Incorrect MIN (date) when minimum value is 0000-00-00
diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test
index 0c2da4ae9f4..18034da9a4c 100644
--- a/mysql-test/t/group_by.test
+++ b/mysql-test/t/group_by.test
@@ -1284,6 +1284,20 @@ FROM t1 GROUP BY a;
 
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#11765254 (58200): Assertion failed: param.sort_length when grouping
+--echo # by functions
+--echo #
+
+SET SQL_BIG_TABLES=1;
+CREATE TABLE t1(a INT);
+INSERT INTO t1 VALUES (0),(0);
+SELECT 1 FROM t1 GROUP BY IF(`a`,'','');
+SELECT 1 FROM t1 GROUP BY TRIM(LEADING RAND() FROM '');
+SELECT 1 FROM t1 GROUP BY SUBSTRING('',SLEEP(0),'');
+SELECT 1 FROM t1 GROUP BY SUBSTRING(SYSDATE() FROM 'K' FOR 'jxW<');
+DROP TABLE t1;
+SET SQL_BIG_TABLES=0;
 
 --echo # End of 5.1 tests
 
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 8a0b37af339..88e9353abe3 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -147,8 +147,6 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   error= 1;
   bzero((char*) &param,sizeof(param));
   param.sort_length= sortlength(thd, sortorder, s_length, &multi_byte_charset);
-  /* filesort cannot handle zero-length records. */
-  DBUG_ASSERT(param.sort_length);
   param.ref_length= table->file->ref_length;
   param.addon_field= 0;
   param.addon_length= 0;
@@ -260,6 +258,9 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   }
   else
   {
+    /* filesort cannot handle zero-length records during merge. */
+    DBUG_ASSERT(param.sort_length != 0);
+
     if (table_sort.buffpek && table_sort.buffpek_len < maxbuffer)
     {
       my_free(table_sort.buffpek);