From cd994f0ee4c67e892d35e9f928b67491b0e264b1 Mon Sep 17 00:00:00 2001
From: Alexey Botchkov <holyfoot@mysql.com>
Date: Thu, 1 Oct 2009 16:33:30 +0500
Subject: [PATCH] Bug#46018      group_concat(distinct ...) uses
 max_heap_table_size for memory allocations     Use min(max_heap_table_size,
 tmp_table_size) instead

per-file comments:
  sql/item_sum.cc
Bug#46018      group_concat(distinct ...) uses max_heap_table_size for memory allocations
    Item_sum_**::setup fixed, so they use ram_limitation() for Unique-s they embed

  sql/item_sum.h
Bug#46018      group_concat(distinct ...) uses max_heap_table_size for memory allocations
   Item_sum::ram_limitation() declared
---
 sql/item_sum.cc | 18 ++++++++++++++----
 sql/item_sum.h  |  1 +
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index 08a48c6ce2f..5ec4ff999ab 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -28,6 +28,17 @@
 #include "mysql_priv.h"
 #include "sql_select.h"
 
+/**
+  Calculate the affordable RAM limit for structures like TREE or Unique
+  used in Item_sum_*
+*/
+
+ulonglong Item_sum::ram_limitation(THD *thd)
+{
+  return min(thd->variables.tmp_table_size,
+      thd->variables.max_heap_table_size);
+}
+
 /**
   Prepare an aggregate function item for checking context conditions.
 
@@ -1045,7 +1056,7 @@ bool Item_sum_distinct::setup(THD *thd)
     are converted to binary representation as well.
   */
   tree= new Unique(simple_raw_key_cmp, &tree_key_length, tree_key_length,
-                   thd->variables.max_heap_table_size);
+                   ram_limitation(thd));
 
   is_evaluated= FALSE;
   DBUG_RETURN(tree == 0);
@@ -2683,8 +2694,7 @@ bool Item_sum_count_distinct::setup(THD *thd)
       }
     }
     DBUG_ASSERT(tree == 0);
-    tree= new Unique(compare_key, cmp_arg, tree_key_length,
-                     thd->variables.max_heap_table_size);
+    tree= new Unique(compare_key,cmp_arg,tree_key_length, ram_limitation(thd));
     /*
       The only time tree_key_length could be 0 is if someone does
       count(distinct) on a char(0) field - stupid thing to do,
@@ -3529,7 +3539,7 @@ bool Item_func_group_concat::setup(THD *thd)
     unique_filter= new Unique(group_concat_key_cmp_with_distinct,
                               (void*)this,
                               tree_key_length,
-                              thd->variables.max_heap_table_size);
+                              ram_limitation(thd));
   
   DBUG_RETURN(FALSE);
 }
diff --git a/sql/item_sum.h b/sql/item_sum.h
index d991327d847..49c3c9892c0 100644
--- a/sql/item_sum.h
+++ b/sql/item_sum.h
@@ -256,6 +256,7 @@ protected:
   Item **orig_args, *tmp_orig_args[2];
   table_map used_tables_cache;
   bool forced_const;
+  static ulonglong ram_limitation(THD *thd);
 
 public: