From 323fdd7ac6e6541a6cc3ab7c48e330805c67d4f3 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Wed, 27 Mar 2013 19:17:32 -0700 Subject: [PATCH] Fixed bug mdev-4311 (bug #68749). This bug was introduced by the patch for WL#3220. If the memory allocated for the tree to store unique elements to be counted is not big enough to include all of them then an external file is used to store the elements. The unique elements are guaranteed not to be nulls. So, when reading them from the file we don't have to care about the null flags of the read values. However, we should remove the flag at the very beginning of the process. If we don't do it and if the last value written into the record buffer for the field whose distinct values needs to be counted happens to be null, then all values read from the file are considered to be nulls and are not counted in. The fix does not remove a possible null flag for the read values. Rather it just counts the values in the same way it was done before WL #3220. --- mysql-test/disabled.def | 1 - mysql-test/r/sum_distinct-big.result | 58 +++++++++++++++++++++++ mysql-test/t/sum_distinct-big.test | 69 ++++++++++++++++++++++++++++ sql/item_sum.cc | 31 ++++++++++++- sql/item_sum.h | 1 + 5 files changed, 158 insertions(+), 2 deletions(-) diff --git a/mysql-test/disabled.def b/mysql-test/disabled.def index 37b4626e6db..3d33be79473 100644 --- a/mysql-test/disabled.def +++ b/mysql-test/disabled.def @@ -13,7 +13,6 @@ tablespace : disabled in MariaDB (no TABLESPACE table attribute) events_time_zone : Test is not predictable as it depends on precise timing. lowercase_table3 : Bug#11762269 2010-06-30 alik main.lowercase_table3 on Mac OSX read_many_rows_innodb : Bug#11748886 2010-11-15 mattiasj report already exists -sum_distinct-big : Bug#11764126 2010-11-15 mattiasj was not tested archive-big : Bug#11817185 2011-03-10 Anitha Disabled since this leads to timeout on Solaris Sparc log_tables-big : Bug#11756699 2010-11-15 mattiasj report already exists mysql_embedded : Bug#12561297 2011-05-14 Anitha Dependent on PB2 changes - eventum#41836 diff --git a/mysql-test/r/sum_distinct-big.result b/mysql-test/r/sum_distinct-big.result index d4933b31f80..2d350826ac8 100644 --- a/mysql-test/r/sum_distinct-big.result +++ b/mysql-test/r/sum_distinct-big.result @@ -1,4 +1,8 @@ DROP TABLE IF EXISTS t1, t2; +set @save_tmp_table_size=@@tmp_table_size; +set @save_max_heap_table_size=@@max_heap_table_size; +set @save_storage_engine=@@storage_engine; +set storage_engine=MYISAM; CREATE TABLE t1 (id INTEGER); CREATE TABLE t2 (id INTEGER); INSERT INTO t1 (id) VALUES (1), (1), (1),(1); @@ -120,3 +124,57 @@ sm 536887296 DROP TABLE t1; DROP TABLE t2; +SET @@tmp_table_size=@save_tmp_table_size; +SET @@max_heap_table_size=@save_max_heap_table_size; +# +# Bug mdev-4311: COUNT(DISTINCT...) requiring a file for Unique +# (bug #68749) +# +set @save_storage_engine=@@storage_engine; +set storage_engine=INNODB; +CREATE TABLE t1 (id INTEGER) ENGINE=InnoDB; +CREATE TABLE t2 (id INTEGER) ENGINE=InnoDB; +INSERT INTO t1 (id) VALUES (1), (1), (1),(1); +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 SELECT id+1 FROM t1; +INSERT INTO t1 SELECT id+2 FROM t1; +INSERT INTO t1 SELECT id+4 FROM t1; +INSERT INTO t1 SELECT id+8 FROM t1; +INSERT INTO t1 SELECT id+16 FROM t1; +INSERT INTO t1 SELECT id+32 FROM t1; +INSERT INTO t1 SELECT id+64 FROM t1; +INSERT INTO t1 SELECT id+128 FROM t1; +INSERT INTO t1 SELECT id+256 FROM t1; +INSERT INTO t1 SELECT id+512 FROM t1; +INSERT INTO t1 SELECT id+1024 FROM t1; +INSERT INTO t1 SELECT id+2048 FROM t1; +INSERT INTO t1 SELECT id+4096 FROM t1; +INSERT INTO t1 SELECT id+8192 FROM t1; +INSERT INTO t2 SELECT id FROM t1 ORDER BY id*rand(); +INSERT INTO t2 VALUE(NULL); +# With default tmp_table_size / max_heap_table_size +SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; +sm +16384 +set @@tmp_table_size=1024*256; +# With reduced tmp_table_size +SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; +sm +16384 +set @@tmp_table_size=@save_tmp_table_size; +SET @@max_heap_table_size=1024*256; +# With reduced max_heap_table_size +SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; +sm +16384 +SET @@max_heap_table_size=@save_max_heap_table_size; +# Back to default tmp_table_size / max_heap_table_size +SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; +sm +16384 +DROP TABLE t1,t2; +set storage_engine=@save_storage_engine; diff --git a/mysql-test/t/sum_distinct-big.test b/mysql-test/t/sum_distinct-big.test index d3710056c9a..fee406ee46d 100644 --- a/mysql-test/t/sum_distinct-big.test +++ b/mysql-test/t/sum_distinct-big.test @@ -3,15 +3,23 @@ # --source include/big_test.inc +--source include/have_innodb.inc --disable_warnings DROP TABLE IF EXISTS t1, t2; --enable_warnings +set @save_tmp_table_size=@@tmp_table_size; +set @save_max_heap_table_size=@@max_heap_table_size; + +set @save_storage_engine=@@storage_engine; + # # Test the case when distinct values doesn't fit in memory and # filesort is used (see uniques.cc:merge_walk) # +set storage_engine=MYISAM; + CREATE TABLE t1 (id INTEGER); CREATE TABLE t2 (id INTEGER); @@ -82,3 +90,64 @@ SELECT SUM(DISTINCT id) sm FROM t2; DROP TABLE t1; DROP TABLE t2; + +SET @@tmp_table_size=@save_tmp_table_size; +SET @@max_heap_table_size=@save_max_heap_table_size; + +--echo # +--echo # Bug mdev-4311: COUNT(DISTINCT...) requiring a file for Unique +--echo # (bug #68749) +--echo # + +set @save_storage_engine=@@storage_engine; +set storage_engine=INNODB; + +CREATE TABLE t1 (id INTEGER) ENGINE=InnoDB; +CREATE TABLE t2 (id INTEGER) ENGINE=InnoDB; + +INSERT INTO t1 (id) VALUES (1), (1), (1),(1); +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 (id) SELECT id FROM t1; +INSERT INTO t1 SELECT id+1 FROM t1; +INSERT INTO t1 SELECT id+2 FROM t1; +INSERT INTO t1 SELECT id+4 FROM t1; +INSERT INTO t1 SELECT id+8 FROM t1; +INSERT INTO t1 SELECT id+16 FROM t1; +INSERT INTO t1 SELECT id+32 FROM t1; +INSERT INTO t1 SELECT id+64 FROM t1; +INSERT INTO t1 SELECT id+128 FROM t1; +INSERT INTO t1 SELECT id+256 FROM t1; +INSERT INTO t1 SELECT id+512 FROM t1; +INSERT INTO t1 SELECT id+1024 FROM t1; +INSERT INTO t1 SELECT id+2048 FROM t1; +INSERT INTO t1 SELECT id+4096 FROM t1; +INSERT INTO t1 SELECT id+8192 FROM t1; + +INSERT INTO t2 SELECT id FROM t1 ORDER BY id*rand(); +INSERT INTO t2 VALUE(NULL); + +--echo # With default tmp_table_size / max_heap_table_size +SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; + +set @@tmp_table_size=1024*256; + +--echo # With reduced tmp_table_size +SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; + +set @@tmp_table_size=@save_tmp_table_size; +SET @@max_heap_table_size=1024*256; + +--echo # With reduced max_heap_table_size +SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; + +SET @@max_heap_table_size=@save_max_heap_table_size; + +--echo # Back to default tmp_table_size / max_heap_table_size +SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; + +DROP TABLE t1,t2; + +set storage_engine=@save_storage_engine; diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 72e0e637d38..3bd00ee828d 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -719,6 +719,14 @@ static int simple_raw_key_cmp(void* arg, const void* key1, const void* key2) } +static int item_sum_distinct_walk_for_count(void *element, + element_count num_of_dups, + void *item) +{ + return ((Aggregator_distinct*) (item))->unique_walk_function_for_count(element); +} + + static int item_sum_distinct_walk(void *element, element_count num_of_dups, void *item) { @@ -1089,7 +1097,12 @@ void Aggregator_distinct::endup() { /* go over the tree of distinct keys and calculate the aggregate value */ use_distinct_values= TRUE; - tree->walk(table, item_sum_distinct_walk, (void*) this); + tree_walk_action func; + if (item_sum->sum_func() == Item_sum::COUNT_DISTINCT_FUNC) + func= item_sum_distinct_walk_for_count; + else + func= item_sum_distinct_walk; + tree->walk(table, func, (void*) this); use_distinct_values= FALSE; } /* prevent consecutive recalculations */ @@ -1466,6 +1479,22 @@ bool Aggregator_distinct::unique_walk_function(void *element) } +/* + A variant of unique_walk_function() that is to be used with Item_sum_count. + + COUNT is a special aggregate function: it doesn't need the values, it only + needs to count them. COUNT needs to know the values are not NULLs, but NULL + values are not put into the Unique, so we don't need to check for NULLs here. +*/ + +bool Aggregator_distinct::unique_walk_function_for_count(void *element) +{ + Item_sum_count *sum= (Item_sum_count *)item_sum; + sum->count++; + return 0; +} + + Aggregator_distinct::~Aggregator_distinct() { if (tree) diff --git a/sql/item_sum.h b/sql/item_sum.h index 40a28d8beae..a954b0f65c1 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -642,6 +642,7 @@ public: virtual bool arg_is_null(); bool unique_walk_function(void *element); + bool unique_walk_function_for_count(void *element); static int composite_key_cmp(void* arg, uchar* key1, uchar* key2); };