From b4b7cf2a95f96892d5ae14608ca52107528b2b7d Mon Sep 17 00:00:00 2001
From: "ramil/ram@mysql.com/ramil.myoffice.izhnet.ru" <>
Date: Fri, 5 Oct 2007 16:33:15 +0500
Subject: [PATCH] Fix for bug #30885: MEMORY returns incorrect data if BTREE
 index is used for NULL lookup

Problem: creating an rb-tree key we store length (2 bytes) before the actual data for
varchar key parts. The fact was missed for NULL key parts, when we set NULL byte and
skip the rest.

Fix: take into account the length of the varchar key parts for NULLs.
---
 heap/hp_hash.c                 | 6 ++++++
 mysql-test/r/heap_btree.result | 8 ++++++++
 mysql-test/t/heap_btree.test   | 9 +++++++++
 3 files changed, 23 insertions(+)

diff --git a/heap/hp_hash.c b/heap/hp_hash.c
index d8eee9c794c..251be1a3fe2 100644
--- a/heap/hp_hash.c
+++ b/heap/hp_hash.c
@@ -808,6 +808,12 @@ uint hp_rb_pack_key(HP_KEYDEF *keydef, uchar *key, const uchar *old,
       if (!(*key++= (char) 1 - *old++))
       {
         k_len-= seg->length;
+        /* 
+          Take into account length (2 bytes) of varchar key parts 
+          stored before the data.
+         */
+        if (seg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART))
+          k_len-= 2;
         continue;
       }
     }
diff --git a/mysql-test/r/heap_btree.result b/mysql-test/r/heap_btree.result
index 91f51a95936..21f5a549529 100644
--- a/mysql-test/r/heap_btree.result
+++ b/mysql-test/r/heap_btree.result
@@ -321,4 +321,12 @@ DROP TABLE t1;
 CREATE TABLE t1 (a INT, UNIQUE USING BTREE(a)) ENGINE=MEMORY;
 INSERT INTO t1 VALUES(NULL),(NULL);
 DROP TABLE t1;
+create table t1(a varchar(255), b varchar(255), 
+key using btree (a,b)) engine=memory;
+insert into t1 values (1, 1), (3, 3), (2, 2), (NULL, 1), (NULL, NULL), (0, 0);
+select * from t1 where a is null;
+a	b
+NULL	NULL
+NULL	1
+drop table t1;
 End of 5.0 tests
diff --git a/mysql-test/t/heap_btree.test b/mysql-test/t/heap_btree.test
index d5a4fb7a734..5a238fc32c8 100644
--- a/mysql-test/t/heap_btree.test
+++ b/mysql-test/t/heap_btree.test
@@ -235,5 +235,14 @@ CREATE TABLE t1 (a INT, UNIQUE USING BTREE(a)) ENGINE=MEMORY;
 INSERT INTO t1 VALUES(NULL),(NULL);
 DROP TABLE t1;
 
+#
+# Bug #30885: MEMORY returns incorrect data if BTREE index is used for NULL lookup
+#
+create table t1(a varchar(255), b varchar(255), 
+                key using btree (a,b)) engine=memory; 
+insert into t1 values (1, 1), (3, 3), (2, 2), (NULL, 1), (NULL, NULL), (0, 0);
+select * from t1 where a is null;
+drop table t1;
+
 --echo End of 5.0 tests