From 04ae1aa9547fdad274127d97a244738c26f7e7e6 Mon Sep 17 00:00:00 2001
From: Alexey Kopytov <Alexey.Kopytov@Sun.com>
Date: Wed, 25 Aug 2010 19:57:53 +0400
Subject: [PATCH 01/10] Bug#55077: Assertion failed: width > 0 && to != ((void
 *)0),            file .\dtoa.c

The assertion failure was correct because the 'width' argument
of my_gcvt() has the signed integer type, whereas the unsigned
value UINT_MAX32 was being passed by the caller
(Field_double::val_str()) leading to a negative width in
my_gcvt().

The following chain of problems was found by further analysis:

1. The display width for a floating point number is calculated
in Field_double::val_str() as either field_length or the
maximum possible length of string representation of a floating
point number, whichever is greater. Since in the bug's test
case field_length is UINT_MAX32, we get the same value as the
display width. This does not make any sense because for numeric
values field_length only matters for ZEROFILL columns,
otherwise it does not make sense to allocate that much memory
just to print a number. Field_float::val_str() has a similar
problem.

2. Even if the above wasn't the case, we would still get a
crash on a slightly different test case when trying to allocate
UINT_MAX32 bytes with String::alloc() because the latter does
not handle such large input values correctly due to alignment
overflows.

3. Even when String::alloc() is fixed to return an error when
an alignment overflow occurs, there is still a problem because
almost no callers check its return value, and
Field_double::val_str() is not an exception (same for
Field_float::val_str()).

4. Even if all of the above wasn't the case, creating a
Field_double object with UINT_MAX32 as its field_length does
not make much sense either, since the .frm code limits it to
MAX_FIELD_CHARLENGTH (255) bytes. Such a beast can only be
created by create_tmp_field_from_item() from an Item with
REAL_RESULT as its result_type() and UINT_MAX32 as its
max_length.

5. For the bug's test case, the above condition (REAL_RESULT
Item with max_length = UINT_MAX32) was a result of
Item_func_if::fix_length_and_dec() "shortcutting" aggregation
of argument types when one of the arguments was a constant
NULL. In this case, the attributes of the aggregated type were
simply copied from the other, non-NULL argument, but max_length
was still calculated as per the general, non-shortcut case, by
choosing the greatest of argument's max_length, which is
obviously not correct.

The patch addresses all of the above problems, even though
fixing the assertion failure for the particular test case would
require only a subset of the above problems to be solved.


client/sql_string.cc:
  Return an error in case of uint32 overflow in alignment.
  Also assert there was no overflow to help find such conditions
  in debug builds, since almost no callers check the return value
  of String::alloc().
mysql-test/r/func_if.result:
  Add a test case for bug #55077.
mysql-test/t/func_if.test:
  Add a test case for bug #55077.
sql/field.cc:
  - Assert we don't operate with fields wider than 255
  (MAX_FIELD_CHARLENGTH) bytes in both Field_float and
  Field_double.
  - Don't take field_length into account when calculating the
  output buffer length.
  - Check the return value of String::alloc()
sql/item_cmpfunc.cc:
  When shortcutting type aggregation, don't take the NULL
  argument's max_length into account.
sql/sql_string.cc:
  Return an error in case of uint32 overflow in alignment.
  Also assert there was no overflow to help find such conditions
  in debug builds, since almost no callers check the return value
  of String::alloc().
---
 client/sql_string.cc        | 10 ++++++++--
 mysql-test/r/func_if.result | 10 ++++++++++
 mysql-test/t/func_if.test   | 12 ++++++++++++
 sql/field.cc                | 21 ++++++++++++++++-----
 sql/item_cmpfunc.cc         | 27 +++++++++++++++------------
 sql/sql_string.cc           | 10 ++++++++--
 6 files changed, 69 insertions(+), 21 deletions(-)

diff --git a/client/sql_string.cc b/client/sql_string.cc
index 6b749409a64..dec6ac94eb2 100644
--- a/client/sql_string.cc
+++ b/client/sql_string.cc
@@ -31,9 +31,12 @@
 ** String functions
 *****************************************************************************/
 
-bool String::real_alloc(uint32 arg_length)
+bool String::real_alloc(uint32 length)
 {
-  arg_length=ALIGN_SIZE(arg_length+1);
+  uint32 arg_length= ALIGN_SIZE(length + 1);
+  DBUG_ASSERT(arg_length > length);
+  if (arg_length <= length)
+    return TRUE;                                 /* Overflow */
   str_length=0;
   if (Alloced_length < arg_length)
   {
@@ -56,6 +59,9 @@ bool String::real_alloc(uint32 arg_length)
 bool String::realloc(uint32 alloc_length)
 {
   uint32 len=ALIGN_SIZE(alloc_length+1);
+  DBUG_ASSERT(len > alloc_length);
+  if (len <= alloc_length)
+    return TRUE;                                 /* Overflow */
   if (Alloced_length < len)
   {
     char *new_ptr;
diff --git a/mysql-test/r/func_if.result b/mysql-test/r/func_if.result
index 955a784f04c..c70589a27b7 100644
--- a/mysql-test/r/func_if.result
+++ b/mysql-test/r/func_if.result
@@ -186,3 +186,13 @@ MAX(IFNULL(CAST(c AS UNSIGNED), 0))
 12345678901234567890
 DROP TABLE t1;
 End of 5.0 tests
+#
+# Bug#55077: Assertion failed: width > 0 && to != ((void *)0), file .\dtoa.c
+# 
+CREATE TABLE t1 (a LONGBLOB, b DOUBLE);
+INSERT INTO t1 VALUES (NULL, 0), (NULL, 1);
+SELECT IF(b, (SELECT a FROM t1 LIMIT 1), b) c FROM t1 GROUP BY c;
+c
+NULL
+0
+DROP TABLE t1;
diff --git a/mysql-test/t/func_if.test b/mysql-test/t/func_if.test
index 4efea8e195e..91f70bb98d7 100644
--- a/mysql-test/t/func_if.test
+++ b/mysql-test/t/func_if.test
@@ -165,3 +165,15 @@ DROP TABLE t1;
 
 
 --echo End of 5.0 tests
+
+
+--echo #
+--echo # Bug#55077: Assertion failed: width > 0 && to != ((void *)0), file .\dtoa.c
+--echo # 
+
+CREATE TABLE t1 (a LONGBLOB, b DOUBLE);
+INSERT INTO t1 VALUES (NULL, 0), (NULL, 1);
+
+SELECT IF(b, (SELECT a FROM t1 LIMIT 1), b) c FROM t1 GROUP BY c;
+
+DROP TABLE t1;
diff --git a/sql/field.cc b/sql/field.cc
index fc55426b177..0e55b624633 100644
--- a/sql/field.cc
+++ b/sql/field.cc
@@ -4189,6 +4189,7 @@ String *Field_float::val_str(String *val_buffer,
 			     String *val_ptr __attribute__((unused)))
 {
   ASSERT_COLUMN_MARKED_FOR_READ;
+  DBUG_ASSERT(field_length <= MAX_FIELD_CHARLENGTH);
   float nr;
 #ifdef WORDS_BIGENDIAN
   if (table->s->db_low_byte_first)
@@ -4199,8 +4200,13 @@ String *Field_float::val_str(String *val_buffer,
 #endif
     memcpy(&nr, ptr, sizeof(nr));
 
-  uint to_length=max(field_length,70);
-  val_buffer->alloc(to_length);
+  uint to_length= 70;
+  if (val_buffer->alloc(to_length))
+  {
+    my_error(ER_OUT_OF_RESOURCES, MYF(0));
+    return val_buffer;
+  }
+
   char *to=(char*) val_buffer->ptr();
   size_t len;
 
@@ -4209,7 +4215,7 @@ String *Field_float::val_str(String *val_buffer,
   else
   {
     /*
-      We are safe here because the buffer length is >= 70, and
+      We are safe here because the buffer length is 70, and
       fabs(float) < 10^39, dec < NOT_FIXED_DEC. So the resulting string
       will be not longer than 69 chars + terminating '\0'.
     */
@@ -4506,6 +4512,7 @@ String *Field_double::val_str(String *val_buffer,
 			      String *val_ptr __attribute__((unused)))
 {
   ASSERT_COLUMN_MARKED_FOR_READ;
+  DBUG_ASSERT(field_length <= MAX_FIELD_CHARLENGTH);
   double nr;
 #ifdef WORDS_BIGENDIAN
   if (table->s->db_low_byte_first)
@@ -4515,9 +4522,13 @@ String *Field_double::val_str(String *val_buffer,
   else
 #endif
     doubleget(nr,ptr);
+  uint to_length= DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE;
+  if (val_buffer->alloc(to_length))
+  {
+    my_error(ER_OUT_OF_RESOURCES, MYF(0));
+    return val_buffer;
+  }
 
-  uint to_length=max(field_length, DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE);
-  val_buffer->alloc(to_length);
   char *to=(char*) val_buffer->ptr();
   size_t len;
 
diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index 641d3726aca..8c0f22b0947 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -2560,27 +2560,30 @@ Item_func_if::fix_length_and_dec()
     cached_result_type= arg2_type;
     collation.set(args[2]->collation.collation);
     cached_field_type= args[2]->field_type();
+    max_length= args[2]->max_length;
+    return;
   }
-  else if (null2)
+
+  if (null2)
   {
     cached_result_type= arg1_type;
     collation.set(args[1]->collation.collation);
     cached_field_type= args[1]->field_type();
+    max_length= args[1]->max_length;
+    return;
+  }
+
+  agg_result_type(&cached_result_type, args + 1, 2);
+  if (cached_result_type == STRING_RESULT)
+  {
+    if (agg_arg_charsets_for_string_result(collation, args + 1, 2))
+      return;
   }
   else
   {
-    agg_result_type(&cached_result_type, args+1, 2);
-    if (cached_result_type == STRING_RESULT)
-    {
-      if (agg_arg_charsets_for_string_result(collation, args + 1, 2))
-        return;
-    }
-    else
-    {
-      collation.set_numeric(); // Number
-    }
-    cached_field_type= agg_field_type(args + 1, 2);
+    collation.set_numeric(); // Number
   }
+  cached_field_type= agg_field_type(args + 1, 2);
 
   uint32 char_length;
   if ((cached_result_type == DECIMAL_RESULT )
diff --git a/sql/sql_string.cc b/sql/sql_string.cc
index 762eebba031..4b7dab243d2 100644
--- a/sql/sql_string.cc
+++ b/sql/sql_string.cc
@@ -31,9 +31,12 @@
 ** String functions
 *****************************************************************************/
 
-bool String::real_alloc(uint32 arg_length)
+bool String::real_alloc(uint32 length)
 {
-  arg_length=ALIGN_SIZE(arg_length+1);
+  uint32 arg_length= ALIGN_SIZE(length + 1);
+  DBUG_ASSERT(arg_length > length);
+  if (arg_length <= length)
+    return TRUE;                                 /* Overflow */
   str_length=0;
   if (Alloced_length < arg_length)
   {
@@ -56,6 +59,9 @@ bool String::real_alloc(uint32 arg_length)
 bool String::realloc(uint32 alloc_length)
 {
   uint32 len=ALIGN_SIZE(alloc_length+1);
+  DBUG_ASSERT(len > alloc_length);
+  if (len <= alloc_length)
+    return TRUE;                                 /* Overflow */
   if (Alloced_length < len)
   {
     char *new_ptr;

From e924c91e57846821d0a1a6d786ddbef412565f7f Mon Sep 17 00:00:00 2001
From: Daniel Fischer <daniel.fischer@oracle.com>
Date: Tue, 31 Aug 2010 13:06:56 +0200
Subject: [PATCH 02/10] Bug#56387 Create initial database in the proper
 location

---
 sql/CMakeLists.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index 224cf67af7f..9d307da364d 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -271,8 +271,7 @@ IF(WIN32 AND MYSQLD_EXECUTABLE)
      COMMAND ${CMAKE_COMMAND}
      ${CONFIG_PARAM} -P ${CMAKE_CURRENT_BINARY_DIR}/create_initial_db.cmake
      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
-     COMMAND  ${CMAKE_COMMAND} -E touch initdb.dep
-     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+     COMMAND  ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep
      DEPENDS mysqld
   )
   ADD_CUSTOM_TARGET(initial_database  

From 02890f0b9082e41b32122e1ef3beb45872eb3b0b Mon Sep 17 00:00:00 2001
From: Alexander Nozdrin <alik@sun.com>
Date: Tue, 31 Aug 2010 17:47:10 +0400
Subject: [PATCH 03/10] Cherry-pick patch for Bug#56120 from
 mysql-5.5-bugfixing.

---
 mysql-test/r/func_group.result | 12 +++++++++++-
 mysql-test/t/func_group.test   | 11 ++++++++++-
 sql/item.cc                    |  9 +++++++--
 3 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/mysql-test/r/func_group.result b/mysql-test/r/func_group.result
index 45c93971852..a9b3e98a26f 100644
--- a/mysql-test/r/func_group.result
+++ b/mysql-test/r/func_group.result
@@ -1715,7 +1715,7 @@ drop table t1;
 #
 End of 5.1 tests
 #
-# Bug#55648: Server crash on MIX/MAX on maximum time value
+# Bug#55648: Server crash on MIN/MAX on maximum time value
 #
 CREATE TABLE t1(c1 TIME NOT NULL);
 INSERT INTO t1 VALUES('837:59:59');
@@ -1725,3 +1725,13 @@ MAX(c1)
 838:59:59
 DROP TABLE t1;
 # End of the bug#55648
+#
+# Bug#56120: Failed assertion on MIN/MAX on negative time value
+#
+CREATE TABLE t1(c1 TIME NOT NULL);
+INSERT INTO t1 VALUES('-00:00:01');
+SELECT MAX(c1),MIN(c1) FROM t1;
+MAX(c1)	MIN(c1)
+-00:00:01	-00:00:01
+DROP TABLE t1;
+# End of the bug#56120
diff --git a/mysql-test/t/func_group.test b/mysql-test/t/func_group.test
index 2914bb15d18..04c52dd32bd 100644
--- a/mysql-test/t/func_group.test
+++ b/mysql-test/t/func_group.test
@@ -1086,7 +1086,7 @@ drop table t1;
 --echo End of 5.1 tests
 
 --echo #
---echo # Bug#55648: Server crash on MIX/MAX on maximum time value
+--echo # Bug#55648: Server crash on MIN/MAX on maximum time value
 --echo #
 CREATE TABLE t1(c1 TIME NOT NULL);
 INSERT INTO t1 VALUES('837:59:59');
@@ -1095,3 +1095,12 @@ SELECT MAX(c1) FROM t1;
 DROP TABLE t1;
 --echo # End of the bug#55648
 
+--echo #
+--echo # Bug#56120: Failed assertion on MIN/MAX on negative time value
+--echo #
+CREATE TABLE t1(c1 TIME NOT NULL);
+INSERT INTO t1 VALUES('-00:00:01');
+SELECT MAX(c1),MIN(c1) FROM t1;
+DROP TABLE t1;
+--echo # End of the bug#56120
+
diff --git a/sql/item.cc b/sql/item.cc
index 24f57342668..427146e11dc 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -7510,9 +7510,14 @@ String *Item_cache_datetime::val_str(String *str)
         return NULL;
       if (cached_field_type == MYSQL_TYPE_TIME)
       {
-        ulonglong time= int_value;
-        DBUG_ASSERT(time <= TIME_MAX_VALUE);
+        longlong time= int_value;
         set_zero_time(&ltime, MYSQL_TIMESTAMP_TIME);
+        if (time < 0)
+        {
+          time= -time;
+          ltime.neg= TRUE;
+        }
+        DBUG_ASSERT(time <= TIME_MAX_VALUE);
         ltime.second= time % 100;
         time/= 100;
         ltime.minute= time % 100;

From 4283a7045845704a6748155a3c1362bf5729b96c Mon Sep 17 00:00:00 2001
From: Alexander Nozdrin <alik@sun.com>
Date: Tue, 31 Aug 2010 17:49:41 +0400
Subject: [PATCH 04/10] Cherry-picking patch for Bug#56137 from
 mysql-5.5-runtime.

------------------------------------------------------------
revno: 3124
revision-id: dlenev@mysql.com-20100831090419-rzr5ktekby2gspm1
parent: alik@sun.com-20100827083901-x4wvtc10u9p7gcs9
committer: Dmitry Lenev <dlenev@mysql.com>
branch nick: mysql-5.5-rt-56137
timestamp: Tue 2010-08-31 13:04:19 +0400
message:
  Bug #56137 "Assertion `thd->lock == 0' failed on upgrading
  from 5.1.50 to 5.5.6".

  Debug builds of the server aborted due to an assertion
  failure when DROP DATABASE statement was run on an
  installation which had outdated or corrupt mysql.proc table.
  Particularly this affected the mysql_upgrade tool which is
  run as part of 5.1 to 5.5 upgrade.

  The problem was that sp_drop_db_routines(), which was invoked
  during dropping of the database, could have returned without
  closing and unlocking mysql.proc table in cases when this
  table was not up-to-date with the current server. As a result
  further attempt to open and lock the mysql.event table, which
  was necessary to complete dropping of the database, ended up
  with an assert.

  This patch solves this problem by ensuring that
  sp_drop_db_routines() always closes mysql.proc table and
  releases metadata locks on it. This is achieved by changing
  open_proc_table_for_update() function to close tables and
  release metadata locks acquired by it in case of failure.
  This step also makes behavior of the latter function
  consistent with behavior of open_proc_table_for_read()/
  open_and_lock_tables().


  Test case for this bug was added to sp-destruct.test.
------------------------------------------------------------
---
 mysql-test/r/sp-destruct.result | 16 ++++++++++++++++
 mysql-test/t/sp-destruct.test   | 30 ++++++++++++++++++++++++++++++
 sql/sp.cc                       |  4 ++++
 3 files changed, 50 insertions(+)

diff --git a/mysql-test/r/sp-destruct.result b/mysql-test/r/sp-destruct.result
index a4933ee9800..32d0e39c5af 100644
--- a/mysql-test/r/sp-destruct.result
+++ b/mysql-test/r/sp-destruct.result
@@ -134,3 +134,19 @@ Warning	1405	Failed to revoke all privileges to dropped routine
 # Restore the procs_priv table
 RENAME TABLE procs_priv_backup TO mysql.procs_priv;
 FLUSH TABLE mysql.procs_priv;
+#
+# Bug #56137 "Assertion `thd->lock == 0' failed on upgrading from
+#             5.1.50 to 5.5.6".
+#
+drop database if exists mysqltest;
+# Backup mysql.proc.
+flush table mysql.proc;
+create database mysqltest;
+# Corrupt mysql.proc to make it unusable by current version of server.
+alter table mysql.proc drop column type;
+# The below statement should not cause assertion failure.
+drop database mysqltest;
+Warnings:
+Error	1547	Column count of mysql.proc is wrong. Expected 20, found 19. The table is probably corrupted
+# Restore mysql.proc.
+drop table mysql.proc;
diff --git a/mysql-test/t/sp-destruct.test b/mysql-test/t/sp-destruct.test
index b7090c01f1e..a5c287e44a8 100644
--- a/mysql-test/t/sp-destruct.test
+++ b/mysql-test/t/sp-destruct.test
@@ -222,3 +222,33 @@ SHOW WARNINGS;
 --echo # Restore the procs_priv table
 RENAME TABLE procs_priv_backup TO mysql.procs_priv;
 FLUSH TABLE mysql.procs_priv;
+
+
+--echo #
+--echo # Bug #56137 "Assertion `thd->lock == 0' failed on upgrading from
+--echo #             5.1.50 to 5.5.6".
+--echo #
+--disable_warnings
+drop database if exists mysqltest;
+--enable_warnings
+--echo # Backup mysql.proc.
+flush table mysql.proc;
+let $MYSQLD_DATADIR= `select @@datadir`;
+--copy_file $MYSQLD_DATADIR/mysql/proc.frm $MYSQLTEST_VARDIR/tmp/proc.frm
+--copy_file $MYSQLD_DATADIR/mysql/proc.MYD $MYSQLTEST_VARDIR/tmp/proc.MYD
+--copy_file $MYSQLD_DATADIR/mysql/proc.MYI $MYSQLTEST_VARDIR/tmp/proc.MYI
+
+create database mysqltest;
+--echo # Corrupt mysql.proc to make it unusable by current version of server.
+alter table mysql.proc drop column type;
+--echo # The below statement should not cause assertion failure.
+drop database mysqltest;
+
+--echo # Restore mysql.proc.
+drop table mysql.proc;
+--copy_file $MYSQLTEST_VARDIR/tmp/proc.frm $MYSQLD_DATADIR/mysql/proc.frm
+--copy_file $MYSQLTEST_VARDIR/tmp/proc.MYD $MYSQLD_DATADIR/mysql/proc.MYD
+--copy_file $MYSQLTEST_VARDIR/tmp/proc.MYI $MYSQLD_DATADIR/mysql/proc.MYI 
+--remove_file $MYSQLTEST_VARDIR/tmp/proc.frm
+--remove_file $MYSQLTEST_VARDIR/tmp/proc.MYD
+--remove_file $MYSQLTEST_VARDIR/tmp/proc.MYI
diff --git a/sql/sp.cc b/sql/sp.cc
index 0265ef45a2a..8821dc9365d 100644
--- a/sql/sp.cc
+++ b/sql/sp.cc
@@ -440,6 +440,7 @@ static TABLE *open_proc_table_for_update(THD *thd)
 {
   TABLE_LIST table_list;
   TABLE *table;
+  MDL_ticket *mdl_savepoint= thd->mdl_context.mdl_savepoint();
   DBUG_ENTER("open_proc_table_for_update");
 
   table_list.init_one_table("mysql", 5, "proc", 4, "proc", TL_WRITE);
@@ -450,6 +451,9 @@ static TABLE *open_proc_table_for_update(THD *thd)
   if (!proc_table_intact.check(table, &proc_table_def))
     DBUG_RETURN(table);
 
+  close_thread_tables(thd);
+  thd->mdl_context.rollback_to_savepoint(mdl_savepoint);
+
   DBUG_RETURN(NULL);
 }
 

From 316e9d63392476de532786f8a6e30d3e6e32b0e4 Mon Sep 17 00:00:00 2001
From: Alexander Nozdrin <alik@sun.com>
Date: Tue, 31 Aug 2010 18:22:03 +0400
Subject: [PATCH 05/10] Cherry-picking patch for Bug#55980. Original changeset:
 ------------------------------------------------------------ revno: 3197
 revision-id: alik@sun.com-20100831135426-h5a4s2w6ih1d8q2x parent:
 magnus.blaudd@sun.com-20100830120632-u3xzy002mdwueli8 committer: Alexander
 Nozdrin <alik@sun.com> branch nick: mysql-5.5-bugfixing timestamp: Tue
 2010-08-31 17:54:26 +0400 message:   Bug#55980 Character sets: supplementary
 character _bin ordering is wrong

  Problem:
  - ORDER BY for utf8mb4_bin, utf16_bin and utf32_bin returned
    results in a wrong order, because old functions
    (supporting only BMP range) were used to handle these collations.
  - Additionally, utf16_bin did not sort supplementary characters
    between U+D700 and U+E000, as WL#1213 specification specified.
------------------------------------------------------------
---
 include/m_ctype.h                      |  5 ++
 mysql-test/include/ctype_filesort2.inc | 16 ++++++
 mysql-test/r/ctype_utf16.result        | 25 +++++++++
 mysql-test/r/ctype_utf32.result        | 25 +++++++++
 mysql-test/r/ctype_utf8mb4.result      | 25 +++++++++
 mysql-test/t/ctype_utf16.test          |  1 +
 mysql-test/t/ctype_utf32.test          |  1 +
 mysql-test/t/ctype_utf8mb4.test        |  1 +
 strings/ctype-ucs2.c                   | 12 ++--
 strings/ctype-utf8.c                   | 76 +++++++++++++++++++++++++-
 10 files changed, 178 insertions(+), 9 deletions(-)
 create mode 100644 mysql-test/include/ctype_filesort2.inc

diff --git a/include/m_ctype.h b/include/m_ctype.h
index 06cbfd779c8..42e8f88cc0e 100644
--- a/include/m_ctype.h
+++ b/include/m_ctype.h
@@ -539,6 +539,11 @@ size_t my_strnxfrm_unicode(CHARSET_INFO *,
                            uchar *dst, size_t dstlen,
                            const uchar *src, size_t srclen);
 
+size_t my_strnxfrm_unicode_full_bin(CHARSET_INFO *,
+                                    uchar *dst, size_t dstlen,
+                                    const uchar *src, size_t srclen);
+size_t  my_strnxfrmlen_unicode_full_bin(CHARSET_INFO *, size_t); 
+
 int my_wildcmp_unicode(CHARSET_INFO *cs,
                        const char *str, const char *str_end,
                        const char *wildstr, const char *wildend,
diff --git a/mysql-test/include/ctype_filesort2.inc b/mysql-test/include/ctype_filesort2.inc
new file mode 100644
index 00000000000..7b09eb482a5
--- /dev/null
+++ b/mysql-test/include/ctype_filesort2.inc
@@ -0,0 +1,16 @@
+#
+# Testing filesort for full Unicode character sets
+# with supplementary characters.
+#
+
+--echo #
+--echo # Bug#55980 Character sets: supplementary character _bin ordering is wrong
+--echo #
+CREATE TABLE t1 AS SELECT REPEAT('a',1) AS a LIMIT 0;
+SHOW CREATE TABLE t1;
+INSERT INTO t1 VALUES (_utf8mb4 0xEFBE9D),(_utf8mb4 0xF0908E84);
+INSERT INTO t1 VALUES (_utf8mb4 0xCE85),(_utf8mb4 0xF4808080);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+ALTER TABLE t1 ADD KEY(a);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+DROP TABLE IF EXISTS t1;
diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result
index c5fd7ef1439..7cee15aecef 100644
--- a/mysql-test/r/ctype_utf16.result
+++ b/mysql-test/r/ctype_utf16.result
@@ -611,6 +611,31 @@ utf16_bin	00610009
 utf16_bin	0061
 utf16_bin	00610020
 drop table t1;
+#
+# Bug#55980 Character sets: supplementary character _bin ordering is wrong
+#
+CREATE TABLE t1 AS SELECT REPEAT('a',1) AS a LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(1) CHARACTER SET utf16 COLLATE utf16_bin NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES (_utf8mb4 0xEFBE9D),(_utf8mb4 0xF0908E84);
+INSERT INTO t1 VALUES (_utf8mb4 0xCE85),(_utf8mb4 0xF4808080);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+0385	CE85
+D800DF84	F0908E84
+DBC0DC00	F4808080
+FF9D	EFBE9D
+ALTER TABLE t1 ADD KEY(a);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+0385	CE85
+D800DF84	F0908E84
+DBC0DC00	F4808080
+FF9D	EFBE9D
+DROP TABLE IF EXISTS t1;
 select @@collation_connection;
 @@collation_connection
 utf16_bin
diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result
index 79e714eab47..94a73093154 100644
--- a/mysql-test/r/ctype_utf32.result
+++ b/mysql-test/r/ctype_utf32.result
@@ -610,6 +610,31 @@ utf32_bin	0000006100000009
 utf32_bin	00000061
 utf32_bin	0000006100000020
 drop table t1;
+#
+# Bug#55980 Character sets: supplementary character _bin ordering is wrong
+#
+CREATE TABLE t1 AS SELECT REPEAT('a',1) AS a LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(1) CHARACTER SET utf32 COLLATE utf32_bin NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES (_utf8mb4 0xEFBE9D),(_utf8mb4 0xF0908E84);
+INSERT INTO t1 VALUES (_utf8mb4 0xCE85),(_utf8mb4 0xF4808080);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+00000385	CE85
+0000FF9D	EFBE9D
+00010384	F0908E84
+00100000	F4808080
+ALTER TABLE t1 ADD KEY(a);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+00000385	CE85
+0000FF9D	EFBE9D
+00010384	F0908E84
+00100000	F4808080
+DROP TABLE IF EXISTS t1;
 select @@collation_connection;
 @@collation_connection
 utf32_bin
diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result
index 454c9d4bfbb..3b9abbc5412 100644
--- a/mysql-test/r/ctype_utf8mb4.result
+++ b/mysql-test/r/ctype_utf8mb4.result
@@ -987,6 +987,31 @@ utf8mb4_bin	6109
 utf8mb4_bin	61
 utf8mb4_bin	6120
 drop table t1;
+#
+# Bug#55980 Character sets: supplementary character _bin ordering is wrong
+#
+CREATE TABLE t1 AS SELECT REPEAT('a',1) AS a LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES (_utf8mb4 0xEFBE9D),(_utf8mb4 0xF0908E84);
+INSERT INTO t1 VALUES (_utf8mb4 0xCE85),(_utf8mb4 0xF4808080);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+CE85	CE85
+EFBE9D	EFBE9D
+F0908E84	F0908E84
+F4808080	F4808080
+ALTER TABLE t1 ADD KEY(a);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+CE85	CE85
+EFBE9D	EFBE9D
+F0908E84	F0908E84
+F4808080	F4808080
+DROP TABLE IF EXISTS t1;
 select @@collation_connection;
 @@collation_connection
 utf8mb4_bin
diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test
index e9c7e569250..ef705474eee 100644
--- a/mysql-test/t/ctype_utf16.test
+++ b/mysql-test/t/ctype_utf16.test
@@ -326,6 +326,7 @@ SET collation_connection='utf16_general_ci';
 SET NAMES latin1;
 SET collation_connection='utf16_bin';
 -- source include/ctype_filesort.inc
+-- source include/ctype_filesort2.inc
 -- source include/ctype_like_escape.inc
 
 #
diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test
index 668b3b033bd..b113403944a 100644
--- a/mysql-test/t/ctype_utf32.test
+++ b/mysql-test/t/ctype_utf32.test
@@ -328,6 +328,7 @@ SET collation_connection='utf32_general_ci';
 SET NAMES latin1;
 SET collation_connection='utf32_bin';
 -- source include/ctype_filesort.inc
+-- source include/ctype_filesort2.inc
 -- source include/ctype_like_escape.inc
 
 #
diff --git a/mysql-test/t/ctype_utf8mb4.test b/mysql-test/t/ctype_utf8mb4.test
index 8fcba92ff47..03696f385b5 100644
--- a/mysql-test/t/ctype_utf8mb4.test
+++ b/mysql-test/t/ctype_utf8mb4.test
@@ -733,6 +733,7 @@ SET collation_connection='utf8mb4_general_ci';
 -- source include/ctype_german.inc
 SET collation_connection='utf8mb4_bin';
 -- source include/ctype_filesort.inc
+-- source include/ctype_filesort2.inc
 -- source include/ctype_like_escape.inc
 
 #
diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c
index d3b0b93a939..ecfac3170d1 100644
--- a/strings/ctype-ucs2.c
+++ b/strings/ctype-ucs2.c
@@ -1469,7 +1469,7 @@ my_strnncoll_utf16_bin(CHARSET_INFO *cs,
     }
     if (s_wc != t_wc)
     {
-      return  s_wc > t_wc ? 1 : -1;
+      return  my_bincmp(s, s + s_res, t, t + t_res);
     }
     
     s+= s_res;
@@ -1511,7 +1511,7 @@ my_strnncollsp_utf16_bin(CHARSET_INFO *cs,
 
     if (s_wc != t_wc)
     {
-      return s_wc > t_wc ? 1 : -1;
+      return my_bincmp(s, s + s_res, t, t + t_res);
     }
 
     s+= s_res;
@@ -1684,8 +1684,8 @@ static MY_COLLATION_HANDLER my_collation_utf16_bin_handler =
   NULL,                /* init */
   my_strnncoll_utf16_bin,
   my_strnncollsp_utf16_bin,
-  my_strnxfrm_unicode,
-  my_strnxfrmlen_simple,
+  my_strnxfrm_unicode_full_bin,
+  my_strnxfrmlen_unicode_full_bin,
   my_like_range_utf16,
   my_wildcmp_utf16_bin,
   my_strcasecmp_mb2_or_mb4,
@@ -2711,8 +2711,8 @@ static MY_COLLATION_HANDLER my_collation_utf32_bin_handler =
   NULL, /* init */
   my_strnncoll_utf32_bin,
   my_strnncollsp_utf32_bin,
-  my_strnxfrm_unicode,
-  my_strnxfrmlen_utf32,
+  my_strnxfrm_unicode_full_bin,
+  my_strnxfrmlen_unicode_full_bin,
   my_like_range_utf32,
   my_wildcmp_utf32_bin,
   my_strcasecmp_mb2_or_mb4,
diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c
index ace39130c12..76fff72290b 100644
--- a/strings/ctype-utf8.c
+++ b/strings/ctype-utf8.c
@@ -1893,7 +1893,13 @@ my_wildcmp_unicode(CHARSET_INFO *cs,
 
 
 /*
-  This function is shared between utf8mb3/utf8mb4/ucs2/utf16/utf32
+  Store sorting weights using 2 bytes per character.
+
+  This function is shared between
+  - utf8mb3_general_ci, utf8_bin, ucs2_general_ci, ucs2_bin
+    which support BMP only (U+0000..U+FFFF).
+  - utf8mb4_general_ci, utf16_general_ci, utf32_general_ci,
+    which map all supplementary characters to weight 0xFFFD.
 */
 size_t
 my_strnxfrm_unicode(CHARSET_INFO *cs,
@@ -1937,6 +1943,70 @@ my_strnxfrm_unicode(CHARSET_INFO *cs,
 }
 
 
+/*
+  Store sorting weights using 3 bytes per character.
+  This function is shared between utf8mb4_bin, utf16_bin, utf32_bin.
+*/
+size_t
+my_strnxfrm_unicode_full_bin(CHARSET_INFO *cs,
+                             uchar *dst, size_t dstlen,
+                             const uchar *src, size_t srclen)
+{
+  my_wc_t wc;
+  uchar *de= dst + dstlen;
+  uchar *de_beg= de - 2; /* The beginning of the last chunk */
+  const uchar *se = src + srclen;
+
+  LINT_INIT(wc);
+  DBUG_ASSERT(src);
+  DBUG_ASSERT(cs->state & MY_CS_BINSORT);
+
+  while (dst < de_beg)
+  {
+    int res;
+    if ((res= cs->cset->mb_wc(cs, &wc, src, se)) <= 0)
+      break;
+    src+= res;
+    if (cs->mbminlen == 2) /* utf16_bin */
+    {
+      /*
+        Reorder code points to weights as follows:
+        U+0000..U+D7FF    -> [00][00][00]..[00][D7][FF] BMP part #1
+        U+10000..U+10FFFF -> [01][00][00]..[10][FF][FF] Supplementary
+        U+E000..U+FFFF    -> [20][E0][00]..[20][FF][FF] BMP part #2
+      */
+      if (wc >= 0xE000 && wc <= 0xFFFF)
+        wc+= 0x200000;
+    }
+    *dst++= (uchar) (wc >> 16);
+    *dst++= (uchar) ((wc >> 8) & 0xFF);
+    *dst++= (uchar) (wc & 0xFF);
+  }
+
+  while (dst < de_beg) /* Fill the tail with keys for space character */
+  {
+    *dst++= 0x00;
+    *dst++= 0x00;
+    *dst++= 0x20;
+  }
+
+  /* Clear the last one or two bytes, if "dstlen" was not divisible by 3 */
+  if (dst < de)
+  {
+    *dst++= 0x00;
+    if (dst < de)
+      *dst= 0x00;
+  }
+
+  return dstlen;
+}
+
+
+size_t
+my_strnxfrmlen_unicode_full_bin(CHARSET_INFO *cs, size_t len)
+{
+  return ((len + 3) / cs->mbmaxlen) * 3;
+}
 #endif /* HAVE_UNIDATA */
 
 
@@ -5067,8 +5137,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_bin_handler =
     NULL,		/* init */
     my_strnncoll_mb_bin,
     my_strnncollsp_mb_bin,
-    my_strnxfrm_unicode,
-    my_strnxfrmlen_utf8mb4,
+    my_strnxfrm_unicode_full_bin,
+    my_strnxfrmlen_unicode_full_bin,
     my_like_range_mb,
     my_wildcmp_mb_bin,
     my_strcasecmp_mb_bin,

From eb129d003e14e93ff3807e2cf4f6591285277f38 Mon Sep 17 00:00:00 2001
From: Alexander Nozdrin <alik@sun.com>
Date: Tue, 31 Aug 2010 18:33:19 +0400
Subject: [PATCH 06/10] Cherry-picking patch for Bug#55846. Original changeset:
 ------------------------------------------------------------ revno: 3186
 revision-id: davi.arnaut@oracle.com-20100831142822-2qhufn3hho4xqr4p parent:
 gshchepa@mysql.com-20100830222201-ie7n43mjy4irlllk committer: Davi Arnaut
 <davi.arnaut@oracle.com> branch nick: mysql-5.5-merge timestamp: Tue
 2010-08-31 11:28:22 -0300 message:   Bug#55846: Link tests fail on Windows -
 my_compiler.h missing

  Make the my_compiler.h header, like my_attribute.h, part of
  the distribution. This is required due to the dependency of
  the former on the latter (which can undefine __attribute__).
------------------------------------------------------------
---
 include/CMakeLists.txt    | 1 +
 include/Makefile.am       | 7 ++++---
 scripts/make_win_bin_dist | 1 +
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt
index aee7561d242..5334fc0f5ae 100644
--- a/include/CMakeLists.txt
+++ b/include/CMakeLists.txt
@@ -54,6 +54,7 @@ SET(HEADERS
   keycache.h
   m_ctype.h
   my_attribute.h 
+  my_compiler.h
   ${HEADERS_GEN_CONFIGURE}
 )
 
diff --git a/include/Makefile.am b/include/Makefile.am
index e30588de065..b0e2402d46c 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -30,8 +30,9 @@ pkginclude_HEADERS =	$(HEADERS_ABI) my_dbug.h m_string.h my_sys.h \
 			decimal.h errmsg.h my_global.h my_net.h \
 			my_getopt.h sslopt-longopts.h my_dir.h \
 			sslopt-vars.h sslopt-case.h sql_common.h keycache.h \
-			m_ctype.h my_attribute.h $(HEADERS_GEN_CONFIGURE) \
-			$(HEADERS_GEN_MAKE) probes_mysql.h probes_mysql_nodtrace.h
+			m_ctype.h my_attribute.h my_compiler.h \
+			$(HEADERS_GEN_CONFIGURE) $(HEADERS_GEN_MAKE) \
+			probes_mysql.h probes_mysql_nodtrace.h
 
 noinst_HEADERS =	lf.h my_bit.h \
 			heap.h my_bitmap.h my_uctype.h password.h \
@@ -45,7 +46,7 @@ noinst_HEADERS =	lf.h my_bit.h \
 			my_user.h my_atomic.h atomic/nolock.h \
 			atomic/rwlock.h atomic/x86-gcc.h atomic/generic-msvc.h \
 			atomic/gcc_builtins.h my_libwrap.h my_stacktrace.h \
-			atomic/solaris.h mysql/innodb_priv.h my_compiler.h
+			atomic/solaris.h mysql/innodb_priv.h
 
 pkgpsiinclude_HEADERS = mysql/psi/psi.h mysql/psi/mysql_thread.h \
 			mysql/psi/mysql_file.h
diff --git a/scripts/make_win_bin_dist b/scripts/make_win_bin_dist
index 4fb57b06d71..aa106a3dfc4 100755
--- a/scripts/make_win_bin_dist
+++ b/scripts/make_win_bin_dist
@@ -260,6 +260,7 @@ cp include/mysql.h \
    include/keycache.h \
    include/m_ctype.h \
    include/my_attribute.h \
+   include/my_compiler.h \
    include/mysqld_error.h \
    include/sql_state.h \
    include/mysqld_ername.h \

From c951c916fc9d54801ae0fce244f14ded10179687 Mon Sep 17 00:00:00 2001
From: Joerg Bruehe <joerg@mysql.com>
Date: Thu, 2 Sep 2010 17:44:25 +0200
Subject: [PATCH 07/10] Check the build mechanisms: Do they support a "-rc"
 suffix?

---
 configure.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.in b/configure.in
index 59d08f0a9e5..eab7c32c068 100644
--- a/configure.in
+++ b/configure.in
@@ -27,7 +27,7 @@ dnl
 dnl When changing the major version number please also check the switch
 dnl statement in mysqlbinlog::check_master_version().  You may also need
 dnl to update version.c in ndb.
-AC_INIT([MySQL Server], [5.5.6-m3], [], [mysql])
+AC_INIT([MySQL Server], [5.5.6-rc], [], [mysql])
 
 AC_CONFIG_SRCDIR([sql/mysqld.cc])
 AC_CANONICAL_SYSTEM

From 9337706bbe1bdfab1d828737a1ae8e3724f6f24e Mon Sep 17 00:00:00 2001
From: Joerg Bruehe <joerg@mysql.com>
Date: Mon, 6 Sep 2010 13:26:23 +0200
Subject: [PATCH 08/10] Fix bug#56547  mysql client interprets commands in
 comments

The problem was not caused by a change in the client,
rather by the tests using the Windows built-in "echo"
and not the one built by MySQL.
This again happened because the binary was missing in the package,
caused by the wrong macro being used to build it in "cmake".
---
 client/CMakeLists.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt
index 771ebf290f1..e39e8d7178a 100644
--- a/client/CMakeLists.txt
+++ b/client/CMakeLists.txt
@@ -64,7 +64,10 @@ MYSQL_ADD_EXECUTABLE(mysqlslap mysqlslap.c)
 SET_SOURCE_FILES_PROPERTIES(mysqlslap.c PROPERTIES COMPILE_FLAGS "-DTHREADS")
 TARGET_LINK_LIBRARIES(mysqlslap mysqlclient)
 
-ADD_EXECUTABLE(echo echo.c)
+# "WIN32" also covers 64 bit. "echo" is used in some files below "mysql-test/".
+IF(WIN32)
+  MYSQL_ADD_EXECUTABLE(echo echo.c)
+ENDIF(WIN32)
 
 SET_TARGET_PROPERTIES (mysqlcheck mysqldump mysqlimport mysql_upgrade mysqlshow mysqlslap 
 PROPERTIES HAS_CXX TRUE)

From f052fa3aad6b5720424f2f07a4e0175901f49b9e Mon Sep 17 00:00:00 2001
From: Dmitry Lenev <dlenev@mysql.com>
Date: Mon, 6 Sep 2010 21:29:02 +0400
Subject: [PATCH 09/10] A temporary workaround for bug #56405 "Deadlock in the
 MDL deadlock detector".

Deadlock could have occurred when workload containing mix
of DML, DDL and FLUSH TABLES statements affecting same
set of tables was executed in heavily concurrent environment.

This deadlock occurred when several connections tried to
perform deadlock detection in metadata locking subsystem.
The first connection started traversing wait-for graph,
encountered sub-graph representing wait for flush, acquired
LOCK_open and dived into sub-graph inspection. When it has
encounterd sub-graph corresponding to wait for metadata lock
and blocked while trying to acquire rd-lock on
MDL_lock::m_rwlock (*) protecting this subgraph, since some
other thread had wr-lock on it. When this wr-lock was released
it could have happened (if there was other pending wr-lock
against this rwlock) that rd-lock from the first connection
was left unsatisfied but at the same time new rd-lock request
from the second connection sneaked in and was satisfied (for
this to be possible second rd- request should come exactly
after wr-lock is released but before pending wr-lock manages
to grab rwlock, which is possible both on Linux and in our
own rwlock implementation). If this second connection
continued traversing wait-for graph and encountered sub-graph
representing wait for flush it tried to acquire LOCK_open
and thus deadlock was created.

This patch tries to workaround this problem but not allowing
deadlock detector to lock LOCK_open mutex if some other thread
doing deadlock detection already owns it and current search
depth is greater than 0. Instead deadlock is reported.

Other possible solutions are either known to have negative
effects on performance or require much more time for proper
implementation and testing.

No test case is provided as this bug is very hard to repeat
in MTR environment but is repeatable with the help of RQG
tests.

sql/mdl.cc:
  Moved Deadlock_detection_visitor::m_current_search_depth to
  parent class to make it available in
  TABLE_SHARE::visit_subgraph().
  Added MDL_wait_for_graph_visitor::abort_traversal() method
  which allows to abort traversal of a wait-for graph and
  report a deadlock.
sql/mdl.h:
  Moved Deadlock_detection_visitor::m_current_search_depth to
  parent class to make it available in
  TABLE_SHARE::visit_subgraph().
  Added MDL_wait_for_graph_visitor::abort_traversal() method
  which allows to abort traversal of a wait-for graph and
  report a deadlock.
sql/sql_base.cc:
  Added dd_owns_lock_open counter and mutex protecting it to
  track number of connections which do deadlock detection and
  own or try to acquire LOCK_open.
sql/sql_base.h:
  Added dd_owns_lock_open counter and mutex protecting it to
  track number of connections which do deadlock detection and
  own or try to acquire LOCK_open.
sql/table.cc:
  Workaround bug #56405 but not allowing MDL deadlock detector
  to lock LOCK_open mutex if some other thread doing deadlock
  detection already owns it and current search depth is greater
  than 0. Instead report deadlock.
---
 sql/mdl.cc      | 34 ++++++++++++++++++++++++----------
 sql/mdl.h       | 16 +++++++++++++++-
 sql/sql_base.cc |  4 ++++
 sql/sql_base.h  |  2 ++
 sql/table.cc    | 33 +++++++++++++++++++++++++++++++++
 5 files changed, 78 insertions(+), 11 deletions(-)

diff --git a/sql/mdl.cc b/sql/mdl.cc
index d53ddcee0c8..aa7c2a4b7f2 100644
--- a/sql/mdl.cc
+++ b/sql/mdl.cc
@@ -124,7 +124,6 @@ public:
   Deadlock_detection_visitor(MDL_context *start_node_arg)
     : m_start_node(start_node_arg),
       m_victim(NULL),
-      m_current_search_depth(0),
       m_found_deadlock(FALSE)
   {}
   virtual bool enter_node(MDL_context *node);
@@ -133,6 +132,8 @@ public:
   virtual bool inspect_edge(MDL_context *dest);
 
   MDL_context *get_victim() const { return m_victim; }
+
+  void abort_traversal(MDL_context *node);
 private:
   /**
     Change the deadlock victim to a new one if it has lower deadlock
@@ -147,13 +148,6 @@ private:
   MDL_context *m_start_node;
   /** If a deadlock is found, the context that identifies the victim. */
   MDL_context *m_victim;
-  /** Set to the 0 at start. Increased whenever
-    we descend into another MDL context (aka traverse to the next
-    wait-for graph node). When MAX_SEARCH_DEPTH is reached, we
-    assume that a deadlock is found, even if we have not found a
-    loop.
-  */
-  uint m_current_search_depth;
   /** TRUE if we found a deadlock. */
   bool m_found_deadlock;
   /**
@@ -187,7 +181,7 @@ private:
 
 bool Deadlock_detection_visitor::enter_node(MDL_context *node)
 {
-  m_found_deadlock= ++m_current_search_depth >= MAX_SEARCH_DEPTH;
+  m_found_deadlock= m_current_search_depth >= MAX_SEARCH_DEPTH;
   if (m_found_deadlock)
   {
     DBUG_ASSERT(! m_victim);
@@ -207,7 +201,6 @@ bool Deadlock_detection_visitor::enter_node(MDL_context *node)
 
 void Deadlock_detection_visitor::leave_node(MDL_context *node)
 {
-  --m_current_search_depth;
   if (m_found_deadlock)
     opt_change_victim_to(node);
 }
@@ -251,6 +244,21 @@ Deadlock_detection_visitor::opt_change_victim_to(MDL_context *new_victim)
 }
 
 
+/**
+  Abort traversal of a wait-for graph and report a deadlock.
+
+  @param node Node which we were about to visit when abort
+              was initiated.
+*/
+
+void Deadlock_detection_visitor::abort_traversal(MDL_context *node)
+{
+  DBUG_ASSERT(! m_victim);
+  m_found_deadlock= TRUE;
+  opt_change_victim_to(node);
+}
+
+
 /**
   Get a bit corresponding to enum_mdl_type value in a granted/waiting bitmaps
   and compatibility matrices.
@@ -2056,8 +2064,13 @@ bool MDL_lock::visit_subgraph(MDL_ticket *waiting_ticket,
     are visiting it but this is OK: in the worst case we might do some
     extra work and one more context might be chosen as a victim.
   */
+  ++gvisitor->m_current_search_depth;
+
   if (gvisitor->enter_node(src_ctx))
+  {
+    --gvisitor->m_current_search_depth;
     goto end;
+  }
 
   /*
     We do a breadth-first search first -- that is, inspect all
@@ -2114,6 +2127,7 @@ bool MDL_lock::visit_subgraph(MDL_ticket *waiting_ticket,
 
 end_leave_node:
   gvisitor->leave_node(src_ctx);
+  --gvisitor->m_current_search_depth;
 
 end:
   mysql_prlock_unlock(&m_rwlock);
diff --git a/sql/mdl.h b/sql/mdl.h
index 7938d833eac..e1d4cf74dd6 100644
--- a/sql/mdl.h
+++ b/sql/mdl.h
@@ -385,7 +385,10 @@ public:
 
   virtual bool inspect_edge(MDL_context *dest) = 0;
   virtual ~MDL_wait_for_graph_visitor();
-  MDL_wait_for_graph_visitor() :m_lock_open_count(0) {}
+  MDL_wait_for_graph_visitor() :m_lock_open_count(0),
+                                m_current_search_depth(0)
+  { }
+  virtual void abort_traversal(MDL_context *node) = 0;
 public:
   /**
    XXX, hack: During deadlock search, we may need to
@@ -396,6 +399,17 @@ public:
    LOCK_open since it has significant performance impacts.
   */
   uint m_lock_open_count;
+  /**
+    Set to the 0 at start. Increased whenever
+    we descend into another MDL context (aka traverse to the next
+    wait-for graph node). When MAX_SEARCH_DEPTH is reached, we
+    assume that a deadlock is found, even if we have not found a
+    loop.
+
+    XXX: This member belongs to this class only temporarily until
+         bug #56405 is fixed.
+  */
+  uint m_current_search_depth;
 };
 
 /**
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index 01ab0b6dec5..b168f561954 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -100,6 +100,8 @@ bool No_such_table_error_handler::safely_trapped_errors()
   TABLE_SHAREs, refresh_version and the table id counter.
 */
 mysql_mutex_t LOCK_open;
+mysql_mutex_t LOCK_dd_owns_lock_open;
+uint dd_owns_lock_open= 0;
 
 #ifdef HAVE_PSI_INTERFACE
 static PSI_mutex_key key_LOCK_open;
@@ -298,6 +300,7 @@ bool table_def_init(void)
   init_tdc_psi_keys();
 #endif
   mysql_mutex_init(key_LOCK_open, &LOCK_open, MY_MUTEX_INIT_FAST);
+  mysql_mutex_init(NULL, &LOCK_dd_owns_lock_open, MY_MUTEX_INIT_FAST);
   oldest_unused_share= &end_of_unused_share;
   end_of_unused_share.prev= &oldest_unused_share;
 
@@ -341,6 +344,7 @@ void table_def_free(void)
     table_def_inited= 0;
     /* Free table definitions. */
     my_hash_free(&table_def_cache);
+    mysql_mutex_destroy(&LOCK_dd_owns_lock_open);
     mysql_mutex_destroy(&LOCK_open);
   }
   DBUG_VOID_RETURN;
diff --git a/sql/sql_base.h b/sql/sql_base.h
index ff935c3fc09..2e4554313e5 100644
--- a/sql/sql_base.h
+++ b/sql/sql_base.h
@@ -71,6 +71,8 @@ enum enum_tdc_remove_table_type {TDC_RT_REMOVE_ALL, TDC_RT_REMOVE_NOT_OWN,
 
 bool check_dup(const char *db, const char *name, TABLE_LIST *tables);
 extern mysql_mutex_t LOCK_open;
+extern mysql_mutex_t LOCK_dd_owns_lock_open;
+extern uint dd_owns_lock_open;
 bool table_cache_init(void);
 void table_cache_free(void);
 bool table_def_init(void);
diff --git a/sql/table.cc b/sql/table.cc
index f08a0aa84ca..030de6719c7 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -3085,7 +3085,30 @@ bool TABLE_SHARE::visit_subgraph(Wait_for_flush *wait_for_flush,
     holding a write-lock on MDL_lock::m_rwlock.
   */
   if (gvisitor->m_lock_open_count++ == 0)
+  {
+    /*
+      To circumvent bug #56405 "Deadlock in the MDL deadlock detector"
+      we don't try to lock LOCK_open mutex if some thread doing
+      deadlock detection already owns it and current search depth is
+      greater than 0. Instead we report a deadlock.
+
+      TODO/FIXME: The proper fix for this bug is to use rwlocks for
+                  protection of table shares/instead of LOCK_open.
+                  Unfortunately it requires more effort/has significant
+                  performance effect.
+    */
+    mysql_mutex_lock(&LOCK_dd_owns_lock_open);
+    if (gvisitor->m_current_search_depth > 0 && dd_owns_lock_open > 0)
+    {
+      mysql_mutex_unlock(&LOCK_dd_owns_lock_open);
+      --gvisitor->m_lock_open_count;
+      gvisitor->abort_traversal(src_ctx);
+      return TRUE;
+    }
+    ++dd_owns_lock_open;
+    mysql_mutex_unlock(&LOCK_dd_owns_lock_open);
     mysql_mutex_lock(&LOCK_open);
+  }
 
   I_P_List_iterator <TABLE, TABLE_share> tables_it(used_tables);
 
@@ -3100,8 +3123,12 @@ bool TABLE_SHARE::visit_subgraph(Wait_for_flush *wait_for_flush,
     goto end;
   }
 
+  ++gvisitor->m_current_search_depth;
   if (gvisitor->enter_node(src_ctx))
+  {
+    --gvisitor->m_current_search_depth;
     goto end;
+  }
 
   while ((table= tables_it++))
   {
@@ -3124,10 +3151,16 @@ bool TABLE_SHARE::visit_subgraph(Wait_for_flush *wait_for_flush,
 
 end_leave_node:
   gvisitor->leave_node(src_ctx);
+  --gvisitor->m_current_search_depth;
 
 end:
   if (gvisitor->m_lock_open_count-- == 1)
+  {
     mysql_mutex_unlock(&LOCK_open);
+    mysql_mutex_lock(&LOCK_dd_owns_lock_open);
+    --dd_owns_lock_open;
+    mysql_mutex_unlock(&LOCK_dd_owns_lock_open);
+  }
 
   return result;
 }

From 9defb07e7dec05cbe899c8a63b7929f7232c37b6 Mon Sep 17 00:00:00 2001
From: Joerg Bruehe <joerg@mysql.com>
Date: Tue, 7 Sep 2010 17:05:16 +0200
Subject: [PATCH 10/10] Fix bug#56574: After installation from RPM, server is
 run under root, not mysql user

The problem was that in the cmake way of building
the variable "MYSQLD_USER" was not set and propagated.
In the script "mysqld_safe" its value is used as the
name of the user who should run the server process.

The fix is to explicitly set this variable to "mysql"
and propagate it in the build process.
It was analyzed and proposed by Jonathan Perkin.
---
 scripts/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt
index 4fc5f3f9b0d..0c9177c79c5 100644
--- a/scripts/CMakeLists.txt
+++ b/scripts/CMakeLists.txt
@@ -139,6 +139,7 @@ ELSE()
 ENDIF()
 
 SET(HOSTNAME "hostname")
+SET(MYSQLD_USER "mysql")
 
 # Required for mysqlbug until autotools are deprecated, once done remove these
 # and expand default cmake variables