From a33bb001445d2576f15437d9392bc70736ab419a Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 8 Nov 2013 14:18:16 +0400 Subject: [PATCH] MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32 --- mysql-test/include/ctype_str_to_date.inc | 12 ++++++++++++ mysql-test/r/ctype_latin1.result | 19 +++++++++++++++++++ mysql-test/r/ctype_sjis.result | 19 +++++++++++++++++++ mysql-test/r/ctype_ucs.result | 19 +++++++++++++++++++ mysql-test/r/ctype_utf8.result | 19 +++++++++++++++++++ mysql-test/t/ctype_latin1.test | 1 + mysql-test/t/ctype_sjis.test | 1 + mysql-test/t/ctype_ucs.test | 1 + mysql-test/t/ctype_utf8.test | 1 + sql/item.cc | 18 ++++++++++++++++++ sql/item.h | 4 ++++ sql/item_timefunc.cc | 18 +++++++++++++++--- sql/item_timefunc.h | 6 +++++- 13 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 mysql-test/include/ctype_str_to_date.inc diff --git a/mysql-test/include/ctype_str_to_date.inc b/mysql-test/include/ctype_str_to_date.inc new file mode 100644 index 00000000000..62adb05b094 --- /dev/null +++ b/mysql-test/include/ctype_str_to_date.inc @@ -0,0 +1,12 @@ +--echo # +--echo # MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32 +--echo # + +SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR)); +SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)); +CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0; +SHOW COLUMNS FROM t1; +INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d'); +SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1; +DROP TABLE t1; + diff --git a/mysql-test/r/ctype_latin1.result b/mysql-test/r/ctype_latin1.result index 66b2cbd8fc4..82b24fcc9ef 100644 --- a/mysql-test/r/ctype_latin1.result +++ b/mysql-test/r/ctype_latin1.result @@ -336,6 +336,25 @@ select c1 as c2h from t1 where c1 like 'ab#_def' escape '#'; c2h ab_def drop table t1; +# +# MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32 +# +SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR)); +@@character_set_connection HEX(CAST(_utf8'÷' AS CHAR)) +latin1 F7 +SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)); +STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)) +2001-01-01 +CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0; +SHOW COLUMNS FROM t1; +Field Type Null Key Default Extra +subject varchar(64) NO +pattern varchar(64) NO +INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d'); +SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1; +HEX(subject) HEX(pattern) STR_TO_DATE(subject, pattern) +32303031F73031F73031 2559F7256DF72564 2001-01-01 00:00:00 +DROP TABLE t1; SET collation_connection='latin1_bin'; create table t1 select repeat('a',4000) a; delete from t1; diff --git a/mysql-test/r/ctype_sjis.result b/mysql-test/r/ctype_sjis.result index 087813d742b..788fd59180e 100644 --- a/mysql-test/r/ctype_sjis.result +++ b/mysql-test/r/ctype_sjis.result @@ -138,6 +138,25 @@ a hex(b) c 3 F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2 4 F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2 DROP TABLE t1; +# +# MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32 +# +SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR)); +@@character_set_connection HEX(CAST(_utf8'÷' AS CHAR)) +sjis 8180 +SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)); +STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)) +2001-01-01 +CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0; +SHOW COLUMNS FROM t1; +Field Type Null Key Default Extra +subject varchar(64) NO +pattern varchar(64) NO +INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d'); +SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1; +HEX(subject) HEX(pattern) STR_TO_DATE(subject, pattern) +323030318180303181803031 25598180256D81802564 2001-01-01 00:00:00 +DROP TABLE t1; SET collation_connection='sjis_bin'; create table t1 select repeat('a',4000) a; delete from t1; diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result index c29074fb226..e6f40b622f0 100644 --- a/mysql-test/r/ctype_ucs.result +++ b/mysql-test/r/ctype_ucs.result @@ -804,6 +804,25 @@ a hex(b) c 3 F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2 4 F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2 DROP TABLE t1; +# +# MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32 +# +SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR)); +@@character_set_connection HEX(CAST(_utf8'÷' AS CHAR)) +ucs2 00F7 +SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)); +STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)) +2001-01-01 +CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0; +SHOW COLUMNS FROM t1; +Field Type Null Key Default Extra +subject varchar(64) NO +pattern varchar(64) NO +INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d'); +SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1; +HEX(subject) HEX(pattern) STR_TO_DATE(subject, pattern) +003200300030003100F70030003100F700300031 0025005900F70025006D00F700250064 2001-01-01 00:00:00 +DROP TABLE t1; SET NAMES latin1; SET collation_connection='ucs2_bin'; create table t1 select repeat('a',4000) a; diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index e61d41bda83..d48ffc81c67 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -976,6 +976,25 @@ ss u,ü ue drop table t1; +# +# MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32 +# +SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR)); +@@character_set_connection HEX(CAST(_utf8'÷' AS CHAR)) +utf8 C3B7 +SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)); +STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR)) +2001-01-01 +CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0; +SHOW COLUMNS FROM t1; +Field Type Null Key Default Extra +subject varchar(64) NO +pattern varchar(64) NO +INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d'); +SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1; +HEX(subject) HEX(pattern) STR_TO_DATE(subject, pattern) +32303031C3B73031C3B73031 2559C3B7256DC3B72564 2001-01-01 00:00:00 +DROP TABLE t1; SET collation_connection='utf8_bin'; create table t1 select repeat('a',4000) a; delete from t1; diff --git a/mysql-test/t/ctype_latin1.test b/mysql-test/t/ctype_latin1.test index a7e6351161b..0c4447faf81 100644 --- a/mysql-test/t/ctype_latin1.test +++ b/mysql-test/t/ctype_latin1.test @@ -82,6 +82,7 @@ select 'a' regexp 'A' collate latin1_bin; SET collation_connection='latin1_swedish_ci'; -- source include/ctype_filesort.inc -- source include/ctype_like_escape.inc +-- source include/ctype_str_to_date.inc SET collation_connection='latin1_bin'; -- source include/ctype_filesort.inc -- source include/ctype_like_escape.inc diff --git a/mysql-test/t/ctype_sjis.test b/mysql-test/t/ctype_sjis.test index 854f9c01a2b..47d2295d620 100644 --- a/mysql-test/t/ctype_sjis.test +++ b/mysql-test/t/ctype_sjis.test @@ -69,6 +69,7 @@ SET collation_connection='sjis_japanese_ci'; -- source include/ctype_innodb_like.inc -- source include/ctype_like_escape.inc -- source include/ctype_like_range_f1f2.inc +-- source include/ctype_str_to_date.inc SET collation_connection='sjis_bin'; -- source include/ctype_filesort.inc -- source include/ctype_innodb_like.inc diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test index 41171b82842..c74f38e6559 100644 --- a/mysql-test/t/ctype_ucs.test +++ b/mysql-test/t/ctype_ucs.test @@ -421,6 +421,7 @@ SET collation_connection='ucs2_general_ci'; -- source include/ctype_like_escape.inc -- source include/ctype_german.inc -- source include/ctype_like_range_f1f2.inc +-- source include/ctype_str_to_date.inc SET NAMES latin1; SET collation_connection='ucs2_bin'; -- source include/ctype_filesort.inc diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index c503e9722f1..6f29379eb8c 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -731,6 +731,7 @@ SET collation_connection='utf8_general_ci'; -- source include/ctype_filesort.inc -- source include/ctype_like_escape.inc -- source include/ctype_german.inc +-- source include/ctype_str_to_date.inc SET collation_connection='utf8_bin'; -- source include/ctype_filesort.inc -- source include/ctype_like_escape.inc diff --git a/sql/item.cc b/sql/item.cc index 803c9fee576..a415464a2f9 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -218,6 +218,24 @@ bool Item::val_bool() } +String *Item::val_str(String *str, String *converter, CHARSET_INFO *cs) +{ + String *res= val_str(str); + if (null_value) + return (String *) 0; + + if (!cs) + return res; + + uint errors; + if ((null_value= converter->copy(res->ptr(), res->length(), + collation.collation, cs, &errors))) + return (String *) 0; + + return converter; +} + + String *Item::val_string_from_real(String *str) { double nr= val_real(); diff --git a/sql/item.h b/sql/item.h index 4f03d588bbc..084e77e4d23 100644 --- a/sql/item.h +++ b/sql/item.h @@ -793,6 +793,10 @@ public: If value is not null null_value flag will be reset to FALSE. */ virtual String *val_str(String *str)=0; + /* + Returns the val_str() value converted to the given character set. + */ + String *val_str(String *str, String *converter, CHARSET_INFO *to); /* Return decimal representation of item with fixed point. diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 6c304946abc..cb60368aafd 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -2980,13 +2980,25 @@ get_date_time_result_type(const char *format, uint length) void Item_func_str_to_date::fix_length_and_dec() { + if (agg_arg_charsets(collation, args, 2, MY_COLL_ALLOW_CONV, 1)) + return; + if (collation.collation->mbminlen > 1) + { +#if MYSQL_VERSION_ID > 50500 + internal_charset= &my_charset_utf8mb4_general_ci; +#else + internal_charset= &my_charset_utf8_general_ci; +#endif + } + cached_field_type= MYSQL_TYPE_DATETIME; decimals= NOT_FIXED_DEC; if ((const_item= args[1]->const_item())) { char format_buff[64]; String format_str(format_buff, sizeof(format_buff), &my_charset_bin); - String *format= args[1]->val_str(&format_str); + String *format= args[1]->val_str(&format_str, &format_converter, + internal_charset); decimals= 0; if (!args[1]->null_value) { @@ -3024,8 +3036,8 @@ bool Item_func_str_to_date::get_date(MYSQL_TIME *ltime, uint fuzzy_date) String format_str(format_buff, sizeof(format_buff), &my_charset_bin), *format; - val= args[0]->val_str(&val_string); - format= args[1]->val_str(&format_str); + val= args[0]->val_str(&val_string, &subject_converter, internal_charset); + format= args[1]->val_str(&format_str, &format_converter, internal_charset); if (args[0]->null_value || args[1]->null_value) goto null_date; diff --git a/sql/item_timefunc.h b/sql/item_timefunc.h index 20365d4f25d..bc048e1ff64 100644 --- a/sql/item_timefunc.h +++ b/sql/item_timefunc.h @@ -998,9 +998,13 @@ class Item_func_str_to_date :public Item_temporal_func enum_field_types cached_field_type; timestamp_type cached_timestamp_type; bool const_item; + String subject_converter; + String format_converter; + CHARSET_INFO *internal_charset; public: Item_func_str_to_date(Item *a, Item *b) - :Item_temporal_func(a, b), const_item(false) + :Item_temporal_func(a, b), const_item(false), + internal_charset(NULL) {} bool get_date(MYSQL_TIME *ltime, uint fuzzy_date); const char *func_name() const { return "str_to_date"; }