From 52d89df285ea94cbd54dba0726599acb4ff91fa4 Mon Sep 17 00:00:00 2001
From: Sergey Glukhov <Sergey.Glukhov@sun.com>
Date: Fri, 26 Mar 2010 09:49:35 +0400
Subject: [PATCH]  Bug#52164 Assertion failed: param.sort_length, file
 .\filesort.cc, line 149 The crash happens because of incorrect max_length
 calculation in QUOTE function(due to overflow). max_length is set to 0 and it
 leads to assert failure. The fix is to cast expression result to ulonglong
 variable and adjust it if the result exceeds MAX_BLOB_WIDTH.

---
 mysql-test/r/func_str.result | 14 ++++++++++++++
 mysql-test/t/func_str.test   | 11 +++++++++++
 sql/item_strfunc.h           |  3 ++-
 3 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/mysql-test/r/func_str.result b/mysql-test/r/func_str.result
index 8cf46e5534e..4268268fabb 100644
--- a/mysql-test/r/func_str.result
+++ b/mysql-test/r/func_str.result
@@ -2587,3 +2587,17 @@ FROM t2 WHERE t2.b = 1 GROUP BY t2.b;
 DECODE((SELECT ENCODE('secret', t1.a) FROM t1,t2 WHERE t1.a = t2.a GROUP BY t1.b LIMIT 1), t2.a)
 secret
 DROP TABLE t1, t2;
+#
+# Bug#52164 Assertion failed: param.sort_length, file .\filesort.cc, line 149
+#
+CREATE TABLE t1 (a LONGBLOB NOT NULL);
+INSERT INTO t1 VALUES (''),('');
+SELECT 1 FROM t1, t1 t2
+ORDER BY QUOTE(t1.a);
+1
+1
+1
+1
+1
+DROP TABLE t1;
+End of 5.1 tests
diff --git a/mysql-test/t/func_str.test b/mysql-test/t/func_str.test
index 8942b0a2faf..3392a41519b 100644
--- a/mysql-test/t/func_str.test
+++ b/mysql-test/t/func_str.test
@@ -1352,3 +1352,14 @@ SELECT DECODE((SELECT ENCODE('secret', t1.a) FROM t1,t2 WHERE t1.a = t2.a GROUP
   FROM t2 WHERE t2.b = 1 GROUP BY t2.b;
 
 DROP TABLE t1, t2;
+
+--echo #
+--echo # Bug#52164 Assertion failed: param.sort_length, file .\filesort.cc, line 149
+--echo #
+CREATE TABLE t1 (a LONGBLOB NOT NULL);
+INSERT INTO t1 VALUES (''),('');
+SELECT 1 FROM t1, t1 t2
+ORDER BY QUOTE(t1.a);
+DROP TABLE t1;
+
+--echo End of 5.1 tests
diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h
index 5799c768162..aedc63164f2 100644
--- a/sql/item_strfunc.h
+++ b/sql/item_strfunc.h
@@ -694,8 +694,9 @@ public:
   String *val_str(String *);
   void fix_length_and_dec()
   {
+    ulonglong max_result_length= (ulonglong) args[0]->max_length * 2 + 2;
+    max_length= min(max_result_length, MAX_BLOB_WIDTH);
     collation.set(args[0]->collation);
-    max_length= args[0]->max_length * 2 + 2;
   }
 };