diff --git a/mysql-test/r/partition.result b/mysql-test/r/partition.result index 0febdbc38d0..b64b0abef0c 100644 --- a/mysql-test/r/partition.result +++ b/mysql-test/r/partition.result @@ -2348,6 +2348,46 @@ TRUNCATE TABLE t1; INSERT INTO t1 VALUES(0); DROP TABLE t1; SET GLOBAL myisam_use_mmap=default; +# +# Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH, +# FILE FILESORT_UTILS.CC +# +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT, +c CHAR(1), +d INT, +KEY (c,d) +) PARTITION BY KEY () PARTITIONS 1; +INSERT INTO t1 VALUES (1,1,'a',1), (2,2,'a',1); +SELECT 1 FROM t1 WHERE 1 IN +(SELECT group_concat(b) +FROM t1 +WHERE c > geomfromtext('point(1 1)') +GROUP BY b +); +1 +1 +1 +DROP TABLE t1; +# +# Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP +# +CREATE TABLE t1 ( +a INT, +b MEDIUMINT, +c VARCHAR(300) CHARACTER SET hp8 COLLATE hp8_bin, +PRIMARY KEY (a,c(299))) +ENGINE=myisam +PARTITION BY LINEAR KEY () PARTITIONS 2; +INSERT INTO t1 VALUES (1,2,'test'), (2,3,'hi'), (4,5,'bye'); +SELECT 1 FROM t1 WHERE b < SOME +( SELECT 1 FROM t1 WHERE a >= 1 +GROUP BY b WITH ROLLUP +HAVING b > geomfromtext("") +); +1 +DROP TABLE t1; End of 5.1 tests # # BUG#55385: UPDATE statement throws an error, but still updates diff --git a/mysql-test/t/partition.test b/mysql-test/t/partition.test index 2a8ba15d3f1..164f6d357e7 100644 --- a/mysql-test/t/partition.test +++ b/mysql-test/t/partition.test @@ -2346,6 +2346,51 @@ INSERT INTO t1 VALUES(0); DROP TABLE t1; SET GLOBAL myisam_use_mmap=default; +--echo # +--echo # Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH, +--echo # FILE FILESORT_UTILS.CC +--echo # + +CREATE TABLE t1 ( + a INT PRIMARY KEY, + b INT, + c CHAR(1), + d INT, + KEY (c,d) +) PARTITION BY KEY () PARTITIONS 1; + +INSERT INTO t1 VALUES (1,1,'a',1), (2,2,'a',1); + +SELECT 1 FROM t1 WHERE 1 IN +(SELECT group_concat(b) + FROM t1 + WHERE c > geomfromtext('point(1 1)') + GROUP BY b +); + +DROP TABLE t1; + +--echo # +--echo # Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP +--echo # + +CREATE TABLE t1 ( + a INT, + b MEDIUMINT, + c VARCHAR(300) CHARACTER SET hp8 COLLATE hp8_bin, + PRIMARY KEY (a,c(299))) +ENGINE=myisam +PARTITION BY LINEAR KEY () PARTITIONS 2; + +INSERT INTO t1 VALUES (1,2,'test'), (2,3,'hi'), (4,5,'bye'); +SELECT 1 FROM t1 WHERE b < SOME +( SELECT 1 FROM t1 WHERE a >= 1 + GROUP BY b WITH ROLLUP + HAVING b > geomfromtext("") +); + +DROP TABLE t1; + --echo End of 5.1 tests --echo # diff --git a/sql/filesort.cc b/sql/filesort.cc index 88e9353abe3..e7740cfdd8a 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -41,8 +41,9 @@ if (my_b_write((file),(uchar*) (from),param->ref_length)) \ /* functions defined in this file */ -static char **make_char_array(char **old_pos, register uint fields, - uint length, myf my_flag); +static size_t char_array_size(uint fields, uint length); +static uchar **make_char_array(uchar **old_pos, uint fields, + uint length, myf my_flag); static uchar *read_buffpek_from_file(IO_CACHE *buffer_file, uint count, uchar *buf); static ha_rows find_all_keys(SORTPARAM *param,SQL_SELECT *select, @@ -224,10 +225,22 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, ulong old_memavl; ulong keys= memavl/(param.rec_length+sizeof(char*)); param.keys=(uint) min(records+1, keys); + + if (table_sort.sort_keys && + table_sort.sort_keys_size != char_array_size(param.keys, + param.rec_length)) + { + my_free(table_sort.sort_keys); + table_sort.sort_keys= NULL; + table_sort.sort_keys_size= 0; + } if ((table_sort.sort_keys= - (uchar **) make_char_array((char **) table_sort.sort_keys, - param.keys, param.rec_length, MYF(0)))) + make_char_array(table_sort.sort_keys, + param.keys, param.rec_length, MYF(0)))) + { + table_sort.sort_keys_size= char_array_size(param.keys, param.rec_length); break; + } old_memavl=memavl; if ((memavl=memavl/4*3) < min_sort_memory && old_memavl > min_sort_memory) memavl= min_sort_memory; @@ -308,6 +321,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, { my_free(sort_keys); table_sort.sort_keys= 0; + table_sort.sort_keys_size= 0; my_free(buffpek); table_sort.buffpek= 0; table_sort.buffpek_len= 0; @@ -369,6 +383,7 @@ void filesort_free_buffers(TABLE *table, bool full) { my_free(table->sort.sort_keys); table->sort.sort_keys= NULL; + table->sort.sort_keys_size= 0; my_free(table->sort.buffpek); table->sort.buffpek= NULL; table->sort.buffpek_len= 0; @@ -382,22 +397,31 @@ void filesort_free_buffers(TABLE *table, bool full) /** Make a array of string pointers. */ -static char **make_char_array(char **old_pos, register uint fields, - uint length, myf my_flag) +static size_t char_array_size(uint fields, uint length) { - register char **pos; - char *char_pos; + return fields * (length + sizeof(uchar*)); +} + + +static uchar **make_char_array(uchar **old_pos, uint fields, + uint length, myf my_flag) +{ + register uchar **pos; + uchar *char_pos; DBUG_ENTER("make_char_array"); DBUG_EXECUTE_IF("make_char_array_fail", DBUG_SET("+d,simulate_out_of_memory");); if (old_pos || - (old_pos= (char**) my_malloc((uint) fields*(length+sizeof(char*)), - my_flag))) + (old_pos= (uchar**) my_malloc(char_array_size(fields, length), my_flag))) { - pos=old_pos; char_pos=((char*) (pos+fields)) -length; - while (fields--) *(pos++) = (char_pos+= length); + pos=old_pos; + char_pos= ((uchar*)(pos+fields)) -length; + while (fields--) + { + *(pos++) = (char_pos+= length); + } } DBUG_RETURN(old_pos); diff --git a/sql/table.h b/sql/table.h index be14270e12b..9a85dd83b13 100644 --- a/sql/table.h +++ b/sql/table.h @@ -298,6 +298,7 @@ typedef struct st_filesort_info { IO_CACHE *io_cache; /* If sorted through filesort */ uchar **sort_keys; /* Buffer for sorting keys */ + size_t sort_keys_size; /* Number of bytes allocated */ uchar *buffpek; /* Buffer for buffpek structures */ uint buffpek_len; /* Max number of buffpeks in the buffer */ uchar *addon_buf; /* Pointer to a buffer if sorted with fields */