From b80641b36cb43206a118ee84fccd90d08fbb8df4 Mon Sep 17 00:00:00 2001
From: unknown <sanja@askmonty.org>
Date: Thu, 8 Sep 2011 16:57:46 +0300
Subject: [PATCH] LP BUG#813418 fix.

The problem was that optimization code did not take into account later feature when instad of NOT before BETWEEN it has negated flag into the Item_func_between inherited from Item_func_neg_opt. So optimizer tried process NOT BETWEEN as BETWEEN.

The patch just switches off the optimisation for NOT BETWEEN as it was before when NOT function was really used.
---
 mysql-test/r/func_group.result | 10 ++++++++++
 mysql-test/t/func_group.test   |  9 +++++++++
 sql/opt_sum.cc                 |  2 ++
 3 files changed, 21 insertions(+)

diff --git a/mysql-test/r/func_group.result b/mysql-test/r/func_group.result
index b90eb2a4c0f..2ff1cd2ec4a 100644
--- a/mysql-test/r/func_group.result
+++ b/mysql-test/r/func_group.result
@@ -1758,3 +1758,13 @@ MAX(f1)
 DROP TABLE t1;
 #
 End of 5.1 tests
+# LP BUG#813418 - incorrect optimisation of max/min by index for
+# negated BETWEEN
+CREATE TABLE t1 (a int, KEY (a));
+INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
+SELECT MAX(a) FROM t1 WHERE a NOT BETWEEN 3 AND 9;
+MAX(a)
+10
+drop table t1;
+#
+End of 5.1 tests
diff --git a/mysql-test/t/func_group.test b/mysql-test/t/func_group.test
index 177a1ca2471..8c2be79ee7d 100644
--- a/mysql-test/t/func_group.test
+++ b/mysql-test/t/func_group.test
@@ -1142,3 +1142,12 @@ DROP TABLE t1;
 --echo #
 --echo End of 5.1 tests
 
+--echo # LP BUG#813418 - incorrect optimisation of max/min by index for
+--echo # negated BETWEEN
+CREATE TABLE t1 (a int, KEY (a));
+INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
+SELECT MAX(a) FROM t1 WHERE a NOT BETWEEN 3 AND 9;
+drop table t1;
+
+--echo #
+--echo End of 5.1 tests
diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc
index 0ed31675f24..eaa3e88abbb 100644
--- a/sql/opt_sum.cc
+++ b/sql/opt_sum.cc
@@ -657,6 +657,8 @@ static bool matching_cond(bool max_fl, TABLE_REF *ref, KEY *keyinfo,
   case Item_func::GE_FUNC:
     break;
   case Item_func::BETWEEN:
+    if (((Item_func_between*) cond)->negated)
+      DBUG_RETURN(FALSE);
     between= 1;
     break;
   case Item_func::MULT_EQUAL_FUNC: