From 8755de32abe3933d0d158801184e7798d39cc44d Mon Sep 17 00:00:00 2001 From: Mikael Ronstrom Date: Mon, 2 Nov 2009 14:49:26 +0100 Subject: [PATCH] Fixed a few bugs in hex string generation, in call to val_str for partition expressions, also made code reusable for DEFAULT handling to fix BUG#48464 by introducing function get_cs_converted_string_value, added partition_utf8 test case for UTF8 outputs --- BUILD/build_mccge.sh | 14 +-- mysql-test/r/partition_column.result | 44 +++----- mysql-test/r/partition_utf8.result | 53 +++++++++ mysql-test/t/partition_column.test | 34 +----- mysql-test/t/partition_utf8.test | 42 +++++++ sql/mysql_priv.h | 1 - sql/sql_partition.cc | 92 +-------------- sql/sql_partition.h | 11 +- sql/sql_show.cc | 162 ++++++++++++++++++++++++++- 9 files changed, 289 insertions(+), 164 deletions(-) create mode 100644 mysql-test/r/partition_utf8.result create mode 100644 mysql-test/t/partition_utf8.test diff --git a/BUILD/build_mccge.sh b/BUILD/build_mccge.sh index 379ca1b2c68..3345ac3dcb5 100755 --- a/BUILD/build_mccge.sh +++ b/BUILD/build_mccge.sh @@ -1274,7 +1274,7 @@ set_bsd_configs() if test "x$fast_flag" != "xno" ; then compiler_flags="$compiler_flags -O3" else - compiler_flags="$compiler_flags -O" + compiler_flags="$compiler_flags -O0" fi set_cc_and_cxx_for_gcc } @@ -1305,7 +1305,7 @@ set_linux_configs() if test "x$fast_flag" != "xno" ; then compiler_flags="$compiler_flags -O2" else - compiler_flags="$compiler_flags -O" + compiler_flags="$compiler_flags -O0" fi # configure will set proper compiler flags for gcc on Linux elif test "x$compiler" = "xicc" ; then @@ -1375,8 +1375,8 @@ set_solaris_configs() LDFLAGS="$LDFLAGS -O2" compiler_flags="$compiler_flags -O2" else - LDFLAGS="$LDFLAGS -O" - compiler_flags="$compiler_flags -O" + LDFLAGS="$LDFLAGS -O0" + compiler_flags="$compiler_flags -O0" fi fi else @@ -1407,7 +1407,7 @@ set_solaris_configs() elif test "x$fast_flag" = "xgeneric" ; then compiler_flags="$compiler_flags -xO2" else - compiler_flags="$compiler_flags -xO" + compiler_flags="$compiler_flags -xO0" fi else #Using SPARC cpu with SunStudio (Forte) compiler @@ -1421,7 +1421,7 @@ set_solaris_configs() elif test "x$fast_flag" = "xgeneric" ; then compiler_flags="$compiler_flags -xO2" else - compiler_flags="$compiler_flags -xO" + compiler_flags="$compiler_flags -xO0" fi fi fi @@ -1452,7 +1452,7 @@ set_macosx_configs() if test "x$fast_flag" != "xno" ; then compiler_flags="$compiler_flags -Os" else - compiler_flags="$compiler_flags -O" + compiler_flags="$compiler_flags -O0" fi set_cc_and_cxx_for_gcc } diff --git a/mysql-test/r/partition_column.result b/mysql-test/r/partition_column.result index 71d159c2f51..8c7d5b4ff77 100644 --- a/mysql-test/r/partition_column.result +++ b/mysql-test/r/partition_column.result @@ -1,18 +1,18 @@ drop table if exists t1; -create table t1 (a varchar(1500), b varchar(1570)) -partition by list columns(a,b) -( partition p0 values in (('a','b'))); -ERROR HY000: The total length of the partitioning fields is too large -create table t1 (a varchar(1023) character set utf8 collate utf8_spanish2_ci) -partition by range columns(a) -( partition p0 values less than ('CZ'), -partition p1 values less than ('CH'), -partition p2 values less than ('D')); -insert into t1 values ('czz'),('chi'),('ci'),('cg'); -select * from t1 where a between 'cg' AND 'ci'; -a -ci -cg +create table t1 (a varchar(5)) +partition by list columns(a) +( partition p0 values in ('\''), + partition p1 values in ('\\'), + partition p2 values in ('\0')); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(5) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY LIST COLUMNS(a) +(PARTITION p0 VALUES IN ('''') ENGINE = MyISAM, + PARTITION p1 VALUES IN ('\\') ENGINE = MyISAM, + PARTITION p2 VALUES IN ('\0') ENGINE = MyISAM) */ drop table t1; set @@sql_mode=allow_invalid_dates; create table t1 (a char, b char, c date) @@ -24,22 +24,6 @@ partition by range columns (a,b,c) ( partition p0 values less than (0,0,'3000-11-31')); ERROR HY000: Partition column values of incorrect type set @@sql_mode=''; -create table t1 (a varchar(2) character set ucs2) -partition by list columns (a) -(partition p0 values in (0x2020), -partition p1 values in ('')); -set names utf8; -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` varchar(2) CHARACTER SET ucs2 DEFAULT NULL -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY LIST COLUMNS(a) -(PARTITION p0 VALUES IN ('†') ENGINE = MyISAM, - PARTITION p1 VALUES IN ('') ENGINE = MyISAM) */ -insert into t1 values (''); -insert into t1 values (_ucs2 0x2020); -drop table t1; create table t1 (a int, b char(10), c varchar(25), d datetime) partition by range columns(a,b,c,d) subpartition by hash (to_seconds(d)) diff --git a/mysql-test/r/partition_utf8.result b/mysql-test/r/partition_utf8.result new file mode 100644 index 00000000000..0fae7bb16b6 --- /dev/null +++ b/mysql-test/r/partition_utf8.result @@ -0,0 +1,53 @@ +set names utf8; +create table t1 (a varchar(2) character set cp1250) +partition by list columns (a) +( partition p0 values in (0x81)); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(2) CHARACTER SET cp1250 DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY LIST COLUMNS(a) +(PARTITION p0 VALUES IN (_cp1250 0x81) ENGINE = MyISAM) */ +drop table t1; +create table t1 (a varchar(2) character set cp1250) +partition by list columns (a) +( partition p0 values in (0x80)); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(2) CHARACTER SET cp1250 DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY LIST COLUMNS(a) +(PARTITION p0 VALUES IN ('€') ENGINE = MyISAM) */ +drop table t1; +create table t1 (a varchar(1500), b varchar(1570)) +partition by list columns(a,b) +( partition p0 values in (('a','b'))); +ERROR HY000: The total length of the partitioning fields is too large +create table t1 (a varchar(1023) character set utf8 collate utf8_spanish2_ci) +partition by range columns(a) +( partition p0 values less than ('CZ'), +partition p1 values less than ('CH'), +partition p2 values less than ('D')); +insert into t1 values ('czz'),('chi'),('ci'),('cg'); +select * from t1 where a between 'cg' AND 'ci'; +a +ci +cg +drop table t1; +create table t1 (a varchar(2) character set ucs2) +partition by list columns (a) +(partition p0 values in (0x2020), +partition p1 values in ('')); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(2) CHARACTER SET ucs2 DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY LIST COLUMNS(a) +(PARTITION p0 VALUES IN ('†') ENGINE = MyISAM, + PARTITION p1 VALUES IN ('') ENGINE = MyISAM) */ +insert into t1 values (''); +insert into t1 values (_ucs2 0x2020); +drop table t1; diff --git a/mysql-test/t/partition_column.test b/mysql-test/t/partition_column.test index b5aea873654..6b62e3576b4 100644 --- a/mysql-test/t/partition_column.test +++ b/mysql-test/t/partition_column.test @@ -8,21 +8,12 @@ drop table if exists t1; --enable_warnings -# -# BUG#48164, too long partition fields causes crash -# ---error ER_PARTITION_FIELDS_TOO_LONG -create table t1 (a varchar(1500), b varchar(1570)) -partition by list columns(a,b) -( partition p0 values in (('a','b'))); - -create table t1 (a varchar(1023) character set utf8 collate utf8_spanish2_ci) -partition by range columns(a) -( partition p0 values less than ('CZ'), - partition p1 values less than ('CH'), - partition p2 values less than ('D')); -insert into t1 values ('czz'),('chi'),('ci'),('cg'); -select * from t1 where a between 'cg' AND 'ci'; +create table t1 (a varchar(5)) +partition by list columns(a) +( partition p0 values in ('\''), + partition p1 values in ('\\'), + partition p2 values in ('\0')); +show create table t1; drop table t1; # @@ -40,19 +31,6 @@ partition by range columns (a,b,c) ( partition p0 values less than (0,0,'3000-11-31')); set @@sql_mode=''; -# -# BUG#48163, Dagger in UCS2 not working as partition value -# -create table t1 (a varchar(2) character set ucs2) -partition by list columns (a) -(partition p0 values in (0x2020), - partition p1 values in ('')); -set names utf8; -show create table t1; -insert into t1 values (''); -insert into t1 values (_ucs2 0x2020); -drop table t1; - --error ER_WRONG_TYPE_COLUMN_VALUE_ERROR create table t1 (a int, b char(10), c varchar(25), d datetime) partition by range columns(a,b,c,d) diff --git a/mysql-test/t/partition_utf8.test b/mysql-test/t/partition_utf8.test new file mode 100644 index 00000000000..d3ad7ba671e --- /dev/null +++ b/mysql-test/t/partition_utf8.test @@ -0,0 +1,42 @@ +# Tests for Column list which requires utf8 output +--source include/have_partition.inc +set names utf8; +create table t1 (a varchar(2) character set cp1250) +partition by list columns (a) +( partition p0 values in (0x81)); +show create table t1; +drop table t1; +create table t1 (a varchar(2) character set cp1250) +partition by list columns (a) +( partition p0 values in (0x80)); +show create table t1; +drop table t1; + +# +# BUG#48164, too long partition fields causes crash +# +--error ER_PARTITION_FIELDS_TOO_LONG +create table t1 (a varchar(1500), b varchar(1570)) +partition by list columns(a,b) +( partition p0 values in (('a','b'))); + +create table t1 (a varchar(1023) character set utf8 collate utf8_spanish2_ci) +partition by range columns(a) +( partition p0 values less than ('CZ'), + partition p1 values less than ('CH'), + partition p2 values less than ('D')); +insert into t1 values ('czz'),('chi'),('ci'),('cg'); +select * from t1 where a between 'cg' AND 'ci'; +drop table t1; + +# +# BUG#48163, Dagger in UCS2 not working as partition value +# +create table t1 (a varchar(2) character set ucs2) +partition by list columns (a) +(partition p0 values in (0x2020), + partition p1 values in ('')); +show create table t1; +insert into t1 values (''); +insert into t1 values (_ucs2 0x2020); +drop table t1; diff --git a/sql/mysql_priv.h b/sql/mysql_priv.h index cff6dd69e15..7825d0c52d4 100644 --- a/sql/mysql_priv.h +++ b/sql/mysql_priv.h @@ -1326,7 +1326,6 @@ void remove_status_vars(SHOW_VAR *list); void init_status_vars(); void free_status_vars(); void reset_status_vars(); - /* information schema */ extern LEX_STRING INFORMATION_SCHEMA_NAME; /* log tables */ diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index df0abf888aa..bec8c0e9b0c 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -2132,92 +2132,6 @@ static Create_field* get_sql_field(char *field_name, } -/* - Convert a string in a given character set to a string which can be - used for FRM file storage in which case use_hex is TRUE and we store - the character constants as hex strings in the character set encoding - their field have. In the case of SHOW CREATE TABLE and the - PARTITIONS information schema table we instead provide utf8 strings - to the user and convert to the utf8 character set. - - SYNOPSIS - get_converted_part_value_from_string() - item Item from which constant comes - res String as provided by val_str after - conversion to character set - field_cs Character set string is encoded in - NULL for INT_RESULT's here - val_conv Out value: The string created - use_hex TRUE => hex string created - FALSE => utf8 constant string created - RETURN VALUES - TRUE Error - FALSE Ok -*/ - -int get_converted_part_value_from_string(Item *item, - String *res, - CHARSET_INFO *field_cs, - String *val_conv, - bool use_hex) -{ - String val; - uint dummy_errors; - uint len, high, low, i; - const char *ptr; - char buf[3]; - - if (item->result_type() == INT_RESULT) - { - longlong value= item->val_int(); - val_conv->set(value, system_charset_info); - return FALSE; - } - if (!res) - { - my_error(ER_PARTITION_FUNCTION_IS_NOT_ALLOWED, MYF(0)); - return TRUE; - } - val_conv->length(0); - if (!field_cs || res->length() == 0) - { - val_conv->append("'"); - if (res->length() != 0) - val_conv->append(*res); - val_conv->append("'"); - return FALSE; - } - if (field_cs && use_hex) - { - val_conv->append("_"); - val_conv->append(field_cs->csname); - val_conv->append(" "); - } - if (use_hex) - { - val_conv->append("0x"); - len= res->length(); - ptr= res->ptr(); - for (i= 0; i < len; i++) - { - high= (*ptr) >> 4; - low= (*ptr) & 0x0F; - buf[0]= _dig_vec_upper[high]; - buf[1]= _dig_vec_upper[low]; - buf[2]= 0; - val_conv->append((const char*)buf); - ptr++; - } - } - else - { - val.copy(res->ptr(), res->length(), field_cs, - system_charset_info, &dummy_errors); - append_unescaped(val_conv, val.ptr(), val.length()); - } - return FALSE; -} - static int add_column_list_values(File fptr, partition_info *part_info, part_elem_value *list_value, HA_CREATE_INFO *create_info, @@ -2311,9 +2225,11 @@ static int add_column_list_values(File fptr, partition_info *part_info, } { String val_conv; + val_conv.set_charset(system_charset_info); res= item_expr->val_str(&str); - if (get_converted_part_value_from_string(item_expr, res, - field_cs, &val_conv, + if (get_cs_converted_part_value_from_string(current_thd, + item_expr, res, + &val_conv, field_cs, (bool)(alter_info != NULL))) return 1; err+= add_string_object(fptr, &val_conv); diff --git a/sql/sql_partition.h b/sql/sql_partition.h index 97b361c4b43..6e1bf8b5728 100644 --- a/sql/sql_partition.h +++ b/sql/sql_partition.h @@ -75,11 +75,12 @@ void get_partition_set(const TABLE *table, uchar *buf, const uint index, const key_range *key_spec, part_id_range *part_spec); uint get_partition_field_store_length(Field *field); -int get_converted_part_value_from_string(Item *item, - String *res, - CHARSET_INFO *cs, - String *val_conv, - bool use_hex); +int get_cs_converted_part_value_from_string(THD *thd, + Item *item, + String *input_str, + String *output_str, + CHARSET_INFO *cs, + bool use_hex); void get_full_part_id_from_key(const TABLE *table, uchar *buf, KEY *key_info, const key_range *key_spec, diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 85964c9bd52..97c0939cb37 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -78,6 +78,12 @@ static TYPELIB grant_types = { sizeof(grant_names)/sizeof(char **), static void store_key_options(THD *thd, String *packet, TABLE *table, KEY *key_info); +static void get_cs_converted_string_value(THD *thd, + String *input_str, + String *output_str, + CHARSET_INFO *cs, + bool use_hex); + static void append_algorithm(TABLE_LIST *table, String *buff); @@ -4795,6 +4801,57 @@ static void collect_partition_expr(List &field_list, String *str) } return; } + + +/* + Convert a string in a given character set to a string which can be + used for FRM file storage in which case use_hex is TRUE and we store + the character constants as hex strings in the character set encoding + their field have. In the case of SHOW CREATE TABLE and the + PARTITIONS information schema table we instead provide utf8 strings + to the user and convert to the utf8 character set. + + SYNOPSIS + get_cs_converted_part_value_from_string() + item Item from which constant comes + input_str String as provided by val_str after + conversion to character set + output_str Out value: The string created + cs Character set string is encoded in + NULL for INT_RESULT's here + use_hex TRUE => hex string created + FALSE => utf8 constant string created + + RETURN VALUES + TRUE Error + FALSE Ok +*/ + +int get_cs_converted_part_value_from_string(THD *thd, + Item *item, + String *input_str, + String *output_str, + CHARSET_INFO *cs, + bool use_hex) +{ + if (item->result_type() == INT_RESULT) + { + longlong value= item->val_int(); + output_str->set(value, system_charset_info); + return FALSE; + } + if (!input_str) + { + my_error(ER_PARTITION_FUNCTION_IS_NOT_ALLOWED, MYF(0)); + return TRUE; + } + get_cs_converted_string_value(thd, + input_str, + output_str, + cs, + use_hex); + return FALSE; +} #endif @@ -4877,7 +4934,8 @@ static void store_schema_partitions_record(THD *thd, TABLE *schema_table, } static int -get_partition_column_description(partition_info *part_info, +get_partition_column_description(THD *thd, + partition_info *part_info, part_elem_value *list_value, String &tmp_str) { @@ -4905,9 +4963,9 @@ get_partition_column_description(partition_info *part_info, DBUG_RETURN(1); } String *res= item->val_str(&str); - if (get_converted_part_value_from_string(item, res, + if (get_cs_converted_part_value_from_string(thd, item, res, &val_conv, part_info->part_field_array[i]->charset(), - &val_conv, FALSE)) + FALSE)) { DBUG_RETURN(1); } @@ -5055,7 +5113,8 @@ static int get_schema_partitions_record(THD *thd, TABLE_LIST *tables, List_iterator list_val_it(part_elem->list_val_list); part_elem_value *list_value= list_val_it++; tmp_str.length(0); - if (get_partition_column_description(part_info, + if (get_partition_column_description(thd, + part_info, list_value, tmp_str)) { @@ -5092,7 +5151,8 @@ static int get_schema_partitions_record(THD *thd, TABLE_LIST *tables, { if (part_info->part_field_list.elements > 1U) tmp_str.append("("); - if (get_partition_column_description(part_info, + if (get_partition_column_description(thd, + part_info, list_value, tmp_str)) { @@ -7231,3 +7291,95 @@ bool show_create_trigger(THD *thd, const sp_name *trg_name) status and client connection will be closed. */ } + +/* + Convert a string in character set in column character set format + to utf8 character set if possible, the utf8 character set string + will later possibly be converted to character set used by client. + Thus we attempt conversion from column character set to both + utf8 and to character set client. + + Examples of strings that should fail conversion to utf8 are unassigned + characters as e.g. 0x81 in cp1250 (Windows character set for for countries + like Czech and Poland). Example of string that should fail conversion to + character set on client (e.g. if this is latin1) is 0x2020 (daggger) in + ucs2. + + If the conversion fails we will as a fall back convert the string to + hex encoded format. The caller of the function can also ask for hex + encoded format of output string unconditionally. + + SYNOPSIS + get_cs_converted_string_value() + thd Thread object + input_str Input string in cs character set + output_str Output string to be produced in utf8 + cs Character set of input string + use_hex Use hex string unconditionally + + + RETURN VALUES + No return value +*/ + +static void get_cs_converted_string_value(THD *thd, + String *input_str, + String *output_str, + CHARSET_INFO *cs, + bool use_hex) +{ + + output_str->length(0); + if (input_str->length() == 0) + { + output_str->append("''"); + return; + } + if (!use_hex) + { + String try_val; + uint try_conv_error= 0; + + try_val.copy(input_str->ptr(), input_str->length(), cs, + thd->variables.character_set_client, &try_conv_error); + if (!try_conv_error) + { + String val; + uint conv_error= 0; + + val.copy(input_str->ptr(), input_str->length(), cs, + system_charset_info, &conv_error); + if (!conv_error) + { + append_unescaped(output_str, val.ptr(), val.length()); + return; + } + } + /* We had a conversion error, use hex encoded string for safety */ + } + { + const uchar *ptr; + uint i, len; + char buf[3]; + + output_str->append("_"); + output_str->append(cs->csname); + output_str->append(" "); + output_str->append("0x"); + len= input_str->length(); + ptr= (uchar*)input_str->ptr(); + for (i= 0; i < len; i++) + { + uint high, low; + + high= (*ptr) >> 4; + low= (*ptr) & 0x0F; + buf[0]= _dig_vec_upper[high]; + buf[1]= _dig_vec_upper[low]; + buf[2]= 0; + output_str->append((const char*)buf); + ptr++; + } + } + return; +}