From 3c6065a270f547d160e11bbcaa653930ec5ff90f Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 6 Dec 2019 18:51:05 +0400 Subject: [PATCH] MDEV-8844 Unreadable control characters printed as is in warnings --- include/m_ctype.h | 16 +++ mysql-test/main/ctype_binary.result | 26 +++++ mysql-test/main/ctype_binary.test | 28 ++++++ mysql-test/main/ctype_latin1.result | 39 ++++++++ mysql-test/main/ctype_latin1.test | 16 +++ mysql-test/main/ctype_ucs.result | 42 ++++++++ mysql-test/main/ctype_ucs.test | 25 +++++ mysql-test/main/ctype_utf16.result | 15 +++ mysql-test/main/ctype_utf16.test | 20 ++++ mysql-test/main/ctype_utf8.result | 74 +++++++++++++- mysql-test/main/ctype_utf8.test | 31 +++++- mysql-test/main/func_str.result | 6 +- mysql-test/main/get_diagnostics.result | 1 + mysql-test/main/get_diagnostics.test | 2 + mysql-test/main/mix2_myisam.result | 2 +- mysql-test/main/mrr_icp_extra.result | 2 +- mysql-test/main/myisam.result | 2 +- mysql-test/main/not_embedded_server.result | 2 +- mysql-test/suite/maria/maria.result | 2 +- .../sys_vars/r/general_log_file_basic.result | 2 +- .../r/slow_query_log_file_basic.result | 2 +- sql/protocol.cc | 12 +++ sql/protocol.h | 1 + sql/sql_error.cc | 97 ++++++++----------- sql/sql_string.cc | 22 +++++ sql/sql_string.h | 9 ++ .../transaction_rollback_delete_delete.result | 2 +- strings/ctype.c | 94 ++++++++++++++++++ 28 files changed, 517 insertions(+), 75 deletions(-) diff --git a/include/m_ctype.h b/include/m_ctype.h index 54e1a166592..60976d437e5 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -45,6 +45,19 @@ extern "C" { #define MY_CS_REPLACEMENT_CHARACTER 0xFFFD +/** + Maximum character length of a string produced by wc_to_printable(). + Note, wc_to_printable() is currently limited to BMP. + One non-printable or non-convertable character can produce a string + with at most 5 characters: \hhhh. + If we ever modify wc_to_printable() to support supplementary characters, + e.g. \+hhhhhh, this constant should be changed to 8. + Note, maximum octet length of a wc_to_printable() result can be calculated + as: (MY_CS_PRINTABLE_CHAR_LENGTH*cs->mbminlen). +*/ +#define MY_CS_PRINTABLE_CHAR_LENGTH 5 + + /* On i386 we store Unicode->CS conversion tables for some character sets using Big-endian order, @@ -740,6 +753,9 @@ int my_wc_mb_bin(CHARSET_INFO *cs,my_wc_t wc, uchar *s, uchar *e); int my_mb_ctype_8bit(CHARSET_INFO *,int *, const uchar *,const uchar *); int my_mb_ctype_mb(CHARSET_INFO *,int *, const uchar *,const uchar *); +int my_wc_to_printable_generic(CHARSET_INFO *cs, my_wc_t wc, + uchar *s, uchar *e); + size_t my_scan_8bit(CHARSET_INFO *cs, const char *b, const char *e, int sq); size_t my_snprintf_8bit(CHARSET_INFO *, char *to, size_t n, diff --git a/mysql-test/main/ctype_binary.result b/mysql-test/main/ctype_binary.result index 118a55cd5ee..77157a4da0a 100644 --- a/mysql-test/main/ctype_binary.result +++ b/mysql-test/main/ctype_binary.result @@ -3174,3 +3174,29 @@ DROP TABLE t1; # # End of 10.1 tests # +# +# Start of 10.5 tests +# +# +# MDEV-8844 Unreadable control characters printed as is in warnings +# +SET NAMES binary; +CREATE TABLE t1 (a VARCHAR(20) CHARACTER SET latin1, UNIQUE(a)); +INSERT INTO t1 VALUES (0x61000162FF); +INSERT INTO t1 VALUES (0x61000162FF); +ERROR 23000: Duplicate entry 'a\0000\0001bÿ' for key 'a' +INSERT IGNORE INTO t1 VALUES (0x61000162FF); +Warnings: +Warning 1062 Duplicate entry 'a\0000\0001bÿ' for key 'a' +DROP TABLE t1; +CREATE TABLE t1 (a VARCHAR(20) CHARACTER SET utf8, UNIQUE(a)); +INSERT INTO t1 VALUES (_latin1 0x61000162FF); +INSERT INTO t1 VALUES (_latin1 0x61000162FF); +ERROR 23000: Duplicate entry 'a\0000\0001bÿ' for key 'a' +INSERT IGNORE INTO t1 VALUES (_latin1 0x61000162FF); +Warnings: +Warning 1062 Duplicate entry 'a\0000\0001bÿ' for key 'a' +DROP TABLE t1; +# +# End of 10.5 tests +# diff --git a/mysql-test/main/ctype_binary.test b/mysql-test/main/ctype_binary.test index 155d8548f77..a466329a48a 100644 --- a/mysql-test/main/ctype_binary.test +++ b/mysql-test/main/ctype_binary.test @@ -77,3 +77,31 @@ DROP TABLE t1; --echo # --echo # End of 10.1 tests --echo # + +--echo # +--echo # Start of 10.5 tests +--echo # + +--echo # +--echo # MDEV-8844 Unreadable control characters printed as is in warnings +--echo # + +SET NAMES binary; +CREATE TABLE t1 (a VARCHAR(20) CHARACTER SET latin1, UNIQUE(a)); +INSERT INTO t1 VALUES (0x61000162FF); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES (0x61000162FF); +INSERT IGNORE INTO t1 VALUES (0x61000162FF); +DROP TABLE t1; + +CREATE TABLE t1 (a VARCHAR(20) CHARACTER SET utf8, UNIQUE(a)); +INSERT INTO t1 VALUES (_latin1 0x61000162FF); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES (_latin1 0x61000162FF); +INSERT IGNORE INTO t1 VALUES (_latin1 0x61000162FF); +DROP TABLE t1; + +--echo # +--echo # End of 10.5 tests +--echo # + diff --git a/mysql-test/main/ctype_latin1.result b/mysql-test/main/ctype_latin1.result index b78a42694cf..81dc03d517d 100644 --- a/mysql-test/main/ctype_latin1.result +++ b/mysql-test/main/ctype_latin1.result @@ -8907,5 +8907,44 @@ t1 CREATE TABLE `t1` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 DROP TABLE t1; # +# MDEV-8844 Unreadable control characters printed as is in warnings +# +SET NAMES latin1; +SELECT CAST(_latin1 0x610062 AS INT); +CAST(_latin1 0x610062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\0000b' +SELECT CAST(_latin1 0x610162 AS INT); +CAST(_latin1 0x610162 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\0001b' +SELECT CAST(_latin1 0x611F62 AS INT); +CAST(_latin1 0x611F62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\001Fb' +SELECT CAST(_latin1 0x617F62 AS INT); +CAST(_latin1 0x617F62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\007Fb' +SELECT CAST(_latin1 0x612062 AS INT); +CAST(_latin1 0x612062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a b' +SELECT CAST(_latin1 0x617E62 AS INT); +CAST(_latin1 0x617E62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a~b' +SELECT CAST(_latin1 0x61FF62 AS INT); +CAST(_latin1 0x61FF62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'ab' +# # End of 10.5 tests # diff --git a/mysql-test/main/ctype_latin1.test b/mysql-test/main/ctype_latin1.test index 97190caf890..d5b62beb140 100644 --- a/mysql-test/main/ctype_latin1.test +++ b/mysql-test/main/ctype_latin1.test @@ -456,6 +456,22 @@ CREATE OR REPLACE TABLE t1 AS SELECT CAST(1 AS BINARY), CAST(@a AS BINARY), CAST SHOW CREATE TABLE t1; DROP TABLE t1; + +--echo # +--echo # MDEV-8844 Unreadable control characters printed as is in warnings +--echo # +SET NAMES latin1; +# control +SELECT CAST(_latin1 0x610062 AS INT); +SELECT CAST(_latin1 0x610162 AS INT); +SELECT CAST(_latin1 0x611F62 AS INT); +SELECT CAST(_latin1 0x617F62 AS INT); +# normal characters +SELECT CAST(_latin1 0x612062 AS INT); +SELECT CAST(_latin1 0x617E62 AS INT); +SELECT CAST(_latin1 0x61FF62 AS INT); + + --echo # --echo # End of 10.5 tests --echo # diff --git a/mysql-test/main/ctype_ucs.result b/mysql-test/main/ctype_ucs.result index 41c3ecab4c4..6f15309211b 100644 --- a/mysql-test/main/ctype_ucs.result +++ b/mysql-test/main/ctype_ucs.result @@ -6431,3 +6431,45 @@ SET NAMES utf8; # # End of 10.4 tests # +# +# Start of 10.5 tests +# +# +# MDEV-8844 Unreadable control characters printed as is in warnings +# +# control +SELECT CAST(_ucs2 0x006100000062 AS INT); +CAST(_ucs2 0x006100000062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\0000b' +SELECT CAST(_ucs2 0x006100010062 AS INT); +CAST(_ucs2 0x006100010062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\0001b' +# surrogate halfs +SELECT CAST(_ucs2 0x0061D8000062 AS INT); +CAST(_ucs2 0x0061D8000062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\D800b' +SELECT CAST(_ucs2 0x0061DFFF0062 AS INT); +CAST(_ucs2 0x0061DFFF0062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\DFFFb' +# normal characters +SELECT CAST(_ucs2 0x0061D7000062 AS INT); +CAST(_ucs2 0x0061D7000062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a휀b' +SELECT CAST(_ucs2 0x0061E0030062 AS INT); +CAST(_ucs2 0x0061E0030062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'ab' +# +# End of 10.5 tests +# diff --git a/mysql-test/main/ctype_ucs.test b/mysql-test/main/ctype_ucs.test index 7a772a092b1..cdc86fa3283 100644 --- a/mysql-test/main/ctype_ucs.test +++ b/mysql-test/main/ctype_ucs.test @@ -1124,3 +1124,28 @@ SET NAMES utf8; --echo # --echo # End of 10.4 tests --echo # + + +--echo # +--echo # Start of 10.5 tests +--echo # + +--echo # +--echo # MDEV-8844 Unreadable control characters printed as is in warnings +--echo # + +--echo # control +SELECT CAST(_ucs2 0x006100000062 AS INT); +SELECT CAST(_ucs2 0x006100010062 AS INT); + +--echo # surrogate halfs +SELECT CAST(_ucs2 0x0061D8000062 AS INT); +SELECT CAST(_ucs2 0x0061DFFF0062 AS INT); + +--echo # normal characters +SELECT CAST(_ucs2 0x0061D7000062 AS INT); +SELECT CAST(_ucs2 0x0061E0030062 AS INT); + +--echo # +--echo # End of 10.5 tests +--echo # diff --git a/mysql-test/main/ctype_utf16.result b/mysql-test/main/ctype_utf16.result index 3c50954242d..24d97d80dbb 100644 --- a/mysql-test/main/ctype_utf16.result +++ b/mysql-test/main/ctype_utf16.result @@ -2814,3 +2814,18 @@ SET STORAGE_ENGINE=Default; # # End of 10.2 tests # +# +# Start of 10.5 tests +# +# +# MDEV-8844 Unreadable control characters printed as is in warnings +# +SET NAMES utf8; +SELECT CAST(_utf16 0x0061D83DDE0E0062 AS INT); +CAST(_utf16 0x0061D83DDE0E0062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a?b' +# +# End of 10.5 tests +# diff --git a/mysql-test/main/ctype_utf16.test b/mysql-test/main/ctype_utf16.test index 0e497b7b7bd..4422f9a6bc7 100644 --- a/mysql-test/main/ctype_utf16.test +++ b/mysql-test/main/ctype_utf16.test @@ -934,3 +934,23 @@ let $coll_pad='utf16_bin'; --echo # --echo # End of 10.2 tests --echo # + + +--echo # +--echo # Start of 10.5 tests +--echo # + +--echo # +--echo # MDEV-8844 Unreadable control characters printed as is in warnings +--echo # + +SET NAMES utf8; +# Make sure surrogate halfs (when a part of a full utf16 character) +# are not escaped and the entire utf16 character consisting of two +# surrogate pairs is replaced to a single question mark. +SELECT CAST(_utf16 0x0061D83DDE0E0062 AS INT); + + +--echo # +--echo # End of 10.5 tests +--echo # diff --git a/mysql-test/main/ctype_utf8.result b/mysql-test/main/ctype_utf8.result index fba7bda67fd..bc87cdeb86e 100644 --- a/mysql-test/main/ctype_utf8.result +++ b/mysql-test/main/ctype_utf8.result @@ -5407,18 +5407,21 @@ DROP TABLE t1; # # Bug#11764503 (Bug#57341) Query in EXPLAIN EXTENDED shows wrong characters # +# Emulate utf8 client erroneously started with --default-character-set=latin1, +# # as in the bug report. EXPLAIN output should still be pretty readable SET NAMES latin1; EXPLAIN EXTENDED SELECT 'abcdÁÂÃÄÅ', _latin1'abcdÁÂÃÄÅ', _utf8'abcdÁÂÃÄÅ' AS u; id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select 'abcdÁÂÃÄÅ' AS `abcdÁÂÃÄÅ`,_latin1'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `abcdÁÂÃÄÅ`,_utf8'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `u` +Note 1003 select 'abcd\0081ÂÃÄÅ' AS `abcd\0081ÂÃÄÅ`,_latin1'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `abcd\0081ÂÃÄÅ`,_utf8'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `u` +# Test normal utf8 SET NAMES utf8; EXPLAIN EXTENDED SELECT 'abcdÁÂÃÄÅ', _latin1'abcdÁÂÃÄÅ', _utf8'abcdÁÂÃÄÅ'; id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select 'abcdÁÂÃÄÅ' AS `abcdÁÂÃÄÅ`,_latin1'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `abcdÁÂÃÄÅ`,_utf8'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `abcdÁÂÃÄÅ` +Note 1003 select 'abcdÁÂÃÄÅ' AS `abcdÁÂÃÄÅ`,_latin1'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `abcdÃ\0081ÂÃÄÅ`,_utf8'abcd\xC3\x81\xC3\x82\xC3\x83\xC3\x84\xC3\x85' AS `abcdÁÂÃÄÅ` # # Bug#11750518 41090: ORDER BY TRUNCATES GROUP_CONCAT RESULT # @@ -11348,5 +11351,72 @@ SELECT uuid()>''; uuid()>'' 1 # +# MDEV-8844 Unreadable control characters printed as is in warnings +# +SET NAMES utf8; +# control, part1 +SELECT CAST(_utf8 0x610062 AS INT); +CAST(_utf8 0x610062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\0000b' +SELECT CAST(_utf8 0x610162 AS INT); +CAST(_utf8 0x610162 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\0001b' +SELECT CAST(_utf8 0x611F62 AS INT); +CAST(_utf8 0x611F62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\001Fb' +# control, part2: U+0080..U+009F +SELECT CAST(_utf8 0x617F62 AS INT); +CAST(_utf8 0x617F62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\007Fb' +SELECT CAST(_utf8 0x61C28062 AS INT); +CAST(_utf8 0x61C28062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\0080b' +SELECT CAST(_utf8 0x61C29F62 AS INT); +CAST(_utf8 0x61C29F62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a\009Fb' +# normal characters +SELECT CAST(_utf8 0x612062 AS INT); +CAST(_utf8 0x612062 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a b' +SELECT CAST(_utf8 0x617E62 AS INT); +CAST(_utf8 0x617E62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a~b' +SELECT CAST(_utf8 0x61C2BF62 AS INT); +CAST(_utf8 0x61C2BF62 AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a¿b' +SELECT CAST(_utf8 'ëëë' AS INT); +CAST(_utf8 'ëëë' AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'ëëë' +SELECT CAST(_utf8 'œœœ' AS INT); +CAST(_utf8 'œœœ' AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'œœœ' +SELECT CAST(_utf8 'яяя' AS INT); +CAST(_utf8 'яяя' AS INT) +0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'яяя' +# # End of 10.5 tests # diff --git a/mysql-test/main/ctype_utf8.test b/mysql-test/main/ctype_utf8.test index 4314a0282e5..ea24561b75c 100644 --- a/mysql-test/main/ctype_utf8.test +++ b/mysql-test/main/ctype_utf8.test @@ -1598,11 +1598,12 @@ DROP TABLE t1; --echo # --echo # Bug#11764503 (Bug#57341) Query in EXPLAIN EXTENDED shows wrong characters --echo # -# Emulate utf8 client erroneously started with --default-character-set=latin1, -# as in the bug report. EXPLAIN output should still be pretty readable + +--echo # Emulate utf8 client erroneously started with --default-character-set=latin1, +--echo # # as in the bug report. EXPLAIN output should still be pretty readable SET NAMES latin1; EXPLAIN EXTENDED SELECT 'abcdÁÂÃÄÅ', _latin1'abcdÁÂÃÄÅ', _utf8'abcdÁÂÃÄÅ' AS u; -# Test normal utf8 +--echo # Test normal utf8 SET NAMES utf8; EXPLAIN EXTENDED SELECT 'abcdÁÂÃÄÅ', _latin1'abcdÁÂÃÄÅ', _utf8'abcdÁÂÃÄÅ'; @@ -2283,6 +2284,30 @@ DROP TABLE t1; SET NAMES utf8 COLLATE utf8_unicode_ci; SELECT uuid()>''; + +--echo # +--echo # MDEV-8844 Unreadable control characters printed as is in warnings +--echo # +SET NAMES utf8; +--echo # control, part1 +SELECT CAST(_utf8 0x610062 AS INT); +SELECT CAST(_utf8 0x610162 AS INT); +SELECT CAST(_utf8 0x611F62 AS INT); + +--echo # control, part2: U+0080..U+009F +SELECT CAST(_utf8 0x617F62 AS INT); +SELECT CAST(_utf8 0x61C28062 AS INT); +SELECT CAST(_utf8 0x61C29F62 AS INT); + +--echo # normal characters +SELECT CAST(_utf8 0x612062 AS INT); +SELECT CAST(_utf8 0x617E62 AS INT); +SELECT CAST(_utf8 0x61C2BF62 AS INT); +SELECT CAST(_utf8 'ëëë' AS INT); +SELECT CAST(_utf8 'œœœ' AS INT); +SELECT CAST(_utf8 'яяя' AS INT); + + --echo # --echo # End of 10.5 tests --echo # diff --git a/mysql-test/main/func_str.result b/mysql-test/main/func_str.result index 5de77f18a24..ba20ba7b3c8 100644 --- a/mysql-test/main/func_str.result +++ b/mysql-test/main/func_str.result @@ -967,17 +967,17 @@ explain extended select length('\n\t\r\b\0\_\%\\'); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select octet_length('\n \r\0\\_\\%\\') AS `length('\n\t\r\b\0\_\%\\')` +Note 1003 select octet_length('\n \r\0008\0\\_\\%\\') AS `length('\n\t\r\b\0\_\%\\')` explain extended select bit_length('\n\t\r\b\0\_\%\\'); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select bit_length('\n \r\0\\_\\%\\') AS `bit_length('\n\t\r\b\0\_\%\\')` +Note 1003 select bit_length('\n \r\0008\0\\_\\%\\') AS `bit_length('\n\t\r\b\0\_\%\\')` explain extended select bit_length('\n\t\r\b\0\_\%\\'); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select bit_length('\n \r\0\\_\\%\\') AS `bit_length('\n\t\r\b\0\_\%\\')` +Note 1003 select bit_length('\n \r\0008\0\\_\\%\\') AS `bit_length('\n\t\r\b\0\_\%\\')` explain extended select concat('monty',' was here ','again'); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used diff --git a/mysql-test/main/get_diagnostics.result b/mysql-test/main/get_diagnostics.result index 6944103c805..ca495d6954d 100644 --- a/mysql-test/main/get_diagnostics.result +++ b/mysql-test/main/get_diagnostics.result @@ -1,3 +1,4 @@ +SET NAMES utf8; # # WL#2111: GET DIAGNOSTICS tests # diff --git a/mysql-test/main/get_diagnostics.test b/mysql-test/main/get_diagnostics.test index 1553eb500b7..4667ec727dd 100644 --- a/mysql-test/main/get_diagnostics.test +++ b/mysql-test/main/get_diagnostics.test @@ -1,3 +1,5 @@ +SET NAMES utf8; + --echo # --echo # WL#2111: GET DIAGNOSTICS tests --echo # diff --git a/mysql-test/main/mix2_myisam.result b/mysql-test/main/mix2_myisam.result index 5acec2616fa..3a4bb5ae0ed 100644 --- a/mysql-test/main/mix2_myisam.result +++ b/mysql-test/main/mix2_myisam.result @@ -1552,7 +1552,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' show warnings; Level Code Message -Error 1062 Duplicate entry 'a' for key 'v_2' +Error 1062 Duplicate entry 'a\0001' for key 'v_2' alter table t1 add key(v); Warnings: Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release diff --git a/mysql-test/main/mrr_icp_extra.result b/mysql-test/main/mrr_icp_extra.result index 176df5cf9d5..f0c284762c6 100644 --- a/mysql-test/main/mrr_icp_extra.result +++ b/mysql-test/main/mrr_icp_extra.result @@ -353,7 +353,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' show warnings; Level Code Message -Error 1062 Duplicate entry 'a' for key 'v_2' +Error 1062 Duplicate entry 'a\0001' for key 'v_2' alter table t1 add key(v); Warnings: Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release diff --git a/mysql-test/main/myisam.result b/mysql-test/main/myisam.result index 7556e64936a..ce2baa03134 100644 --- a/mysql-test/main/myisam.result +++ b/mysql-test/main/myisam.result @@ -1268,7 +1268,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' show warnings; Level Code Message -Error 1062 Duplicate entry 'a' for key 'v_2' +Error 1062 Duplicate entry 'a\0001' for key 'v_2' alter table t1 add key(v); Warnings: Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release diff --git a/mysql-test/main/not_embedded_server.result b/mysql-test/main/not_embedded_server.result index bc794ce48c8..0b952f93ad8 100644 --- a/mysql-test/main/not_embedded_server.result +++ b/mysql-test/main/not_embedded_server.result @@ -68,7 +68,7 @@ ERROR HY000: Operation CREATE USER failed for 'user\"s_12601974'@'localhost' DROP USER 'user\"s_12601974'@'localhost'; CREATE USER 'user\bs_12601974'@'localhost'; CREATE USER 'user\bs_12601974'@'localhost'; -ERROR HY000: Operation CREATE USER failed for 'users_12601974'@'localhost' +ERROR HY000: Operation CREATE USER failed for 'user\0008s_12601974'@'localhost' DROP USER 'user\bs_12601974'@'localhost'; CREATE USER 'user\ns_12601974'@'localhost'; CREATE USER 'user\ns_12601974'@'localhost'; diff --git a/mysql-test/suite/maria/maria.result b/mysql-test/suite/maria/maria.result index 908df15aa0d..9b0307d4968 100644 --- a/mysql-test/suite/maria/maria.result +++ b/mysql-test/suite/maria/maria.result @@ -1156,7 +1156,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' show warnings; Level Code Message -Error 1062 Duplicate entry 'a' for key 'v_2' +Error 1062 Duplicate entry 'a\0001' for key 'v_2' alter table t1 add key(v); Warnings: Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release diff --git a/mysql-test/suite/sys_vars/r/general_log_file_basic.result b/mysql-test/suite/sys_vars/r/general_log_file_basic.result index 6a80749453b..e04f7ee37a4 100644 --- a/mysql-test/suite/sys_vars/r/general_log_file_basic.result +++ b/mysql-test/suite/sys_vars/r/general_log_file_basic.result @@ -19,7 +19,7 @@ ERROR 42000: Variable 'general_log_file' can't be set to the value of '/tmp/my.c SET @@global.general_log_file = '.my.cnf'; ERROR 42000: Variable 'general_log_file' can't be set to the value of '.my.cnf' SET @@global.general_log_file = 'my.cnf\0foo'; -ERROR 42000: Variable 'general_log_file' can't be set to the value of 'my.cnf' +ERROR 42000: Variable 'general_log_file' can't be set to the value of 'my.cnf\0000foo' SET @@global.general_log_file = 'my.ini'; ERROR 42000: Variable 'general_log_file' can't be set to the value of 'my.ini' '#----------------------FN_DYNVARS_004_03------------------------#' diff --git a/mysql-test/suite/sys_vars/r/slow_query_log_file_basic.result b/mysql-test/suite/sys_vars/r/slow_query_log_file_basic.result index 74cf0da9338..47bc780c96a 100644 --- a/mysql-test/suite/sys_vars/r/slow_query_log_file_basic.result +++ b/mysql-test/suite/sys_vars/r/slow_query_log_file_basic.result @@ -16,7 +16,7 @@ ERROR 42000: Variable 'slow_query_log_file' can't be set to the value of '/tmp/m SET @@global.general_log_file = '.my.cnf'; ERROR 42000: Variable 'general_log_file' can't be set to the value of '.my.cnf' SET @@global.general_log_file = 'my.cnf\0foo'; -ERROR 42000: Variable 'general_log_file' can't be set to the value of 'my.cnf' +ERROR 42000: Variable 'general_log_file' can't be set to the value of 'my.cnf\0000foo' SET @@global.general_log_file = 'my.ini'; ERROR 42000: Variable 'general_log_file' can't be set to the value of 'my.ini' '#----------------------FN_DYNVARS_004_03------------------------#' diff --git a/sql/protocol.cc b/sql/protocol.cc index 7e63fcf81a4..30e4cc6c214 100644 --- a/sql/protocol.cc +++ b/sql/protocol.cc @@ -1143,6 +1143,18 @@ bool Protocol::store_string_aux(const char *from, size_t length, } +bool Protocol::store_warning(const char *from, size_t length) +{ + BinaryStringBuffer tmp; + CHARSET_INFO *cs= thd->variables.character_set_results; + if (cs == &my_charset_bin) + cs= system_charset_info; + if (tmp.copy_printable_hhhh(cs, system_charset_info, from, length)) + return net_store_data((const uchar*)"", 0); + return net_store_data((const uchar *) tmp.ptr(), tmp.length()); +} + + bool Protocol_text::store(const char *from, size_t length, CHARSET_INFO *fromcs, CHARSET_INFO *tocs) { diff --git a/sql/protocol.h b/sql/protocol.h index c5e525cac8c..661ca11d3a1 100644 --- a/sql/protocol.h +++ b/sql/protocol.h @@ -90,6 +90,7 @@ public: bool store(I_List *str_list); bool store(const char *from, CHARSET_INFO *cs); + bool store_warning(const char *from, size_t length); String *storage_packet() { return packet; } inline void free() { packet->free(); } virtual bool write(); diff --git a/sql/sql_error.cc b/sql/sql_error.cc index 92a1adb8ab7..7e6ffb67b94 100644 --- a/sql/sql_error.cc +++ b/sql/sql_error.cc @@ -822,9 +822,8 @@ bool mysqld_show_warnings(THD *thd, ulong levels_to_show) warning_level_names[err->get_level()].length, system_charset_info); protocol->store((uint32) err->get_sql_errno()); - protocol->store(err->get_message_text(), - err->get_message_octet_length(), - system_charset_info); + protocol->store_warning(err->get_message_text(), + err->get_message_octet_length()); if (protocol->write()) DBUG_RETURN(TRUE); } @@ -836,6 +835,26 @@ bool mysqld_show_warnings(THD *thd, ulong levels_to_show) } +/** + This replaces U+0000 to '\0000', so the result error message string: + - is a good null-terminated string + - presents the entire data + For example: + SELECT CAST(_latin1 0x610062 AS SIGNED); + returns a warning: + Truncated incorrect INTEGER value: 'a\0000b' + Notice, the 0x00 byte is replaced to a 5-byte long string '\0000', + while 'a' and 'b' are printed as is. +*/ +extern "C" int my_wc_mb_utf8_null_terminated(CHARSET_INFO *cs, + my_wc_t wc, uchar *r, uchar *e) +{ + return wc == '\0' ? + my_wc_to_printable_generic(cs, wc, r, e) : + my_charset_utf8mb3_handler.wc_mb(cs, wc, r, e); +} + + /** Convert value for dispatch to error message(see WL#751). @@ -894,8 +913,11 @@ char *err_conv(char *buff, uint to_length, const char *from, else { uint errors; - res= copy_and_convert(to, to_length, system_charset_info, - from, from_length, from_cs, &errors); + res= my_convert_using_func(to, to_length, system_charset_info, + my_wc_mb_utf8_null_terminated, + from, from_length, from_cs, + from_cs->cset->mb_wc, + &errors); to[res]= 0; } return buff; @@ -921,64 +943,21 @@ size_t convert_error_message(char *to, size_t to_length, CHARSET_INFO *to_cs, const char *from, size_t from_length, CHARSET_INFO *from_cs, uint *errors) { - int cnvres; - my_wc_t wc; - const uchar *from_end= (const uchar*) from+from_length; - char *to_start= to; - uchar *to_end; - my_charset_conv_mb_wc mb_wc= from_cs->cset->mb_wc; - my_charset_conv_wc_mb wc_mb; - uint error_count= 0; - size_t length; - DBUG_ASSERT(to_length > 0); /* Make room for the null terminator. */ to_length--; - to_end= (uchar*) (to + to_length); - if (!to_cs || from_cs == to_cs || to_cs == &my_charset_bin) - { - length= MY_MIN(to_length, from_length); - memmove(to, from, length); - to[length]= 0; - return length; - } - - wc_mb= to_cs->cset->wc_mb; - while (1) - { - if ((cnvres= (*mb_wc)(from_cs, &wc, (uchar*) from, from_end)) > 0) - { - if (!wc) - break; - from+= cnvres; - } - else if (cnvres == MY_CS_ILSEQ) - { - wc= (ulong) (uchar) *from; - from+=1; - } - else - break; - - if ((cnvres= (*wc_mb)(to_cs, wc, (uchar*) to, to_end)) > 0) - to+= cnvres; - else if (cnvres == MY_CS_ILUNI) - { - length= (wc <= 0xFFFF) ? 6/* '\1234' format*/ : 9 /* '\+123456' format*/; - if ((uchar*)(to + length) >= to_end) - break; - cnvres= (int)my_snprintf(to, 9, - (wc <= 0xFFFF) ? "\\%04X" : "\\+%06X", (uint) wc); - to+= cnvres; - } - else - break; - } - - *to= 0; - *errors= error_count; - return (size_t) (to - to_start); + if (to_cs == &my_charset_bin) + to_cs= system_charset_info; + uint32 cnv_length= my_convert_using_func(to, to_length, + to_cs, + my_wc_to_printable_generic, + from, from_length, + from_cs, from_cs->cset->mb_wc, + errors); + DBUG_ASSERT(to_length >= cnv_length); + to[cnv_length]= '\0'; + return cnv_length; } diff --git a/sql/sql_string.cc b/sql/sql_string.cc index 483eb4fcbec..024f9330c84 100644 --- a/sql/sql_string.cc +++ b/sql/sql_string.cc @@ -777,6 +777,28 @@ void Static_binary_string::qs_append(ulonglong i) str_length+= (int) (end-buff); } + +bool Binary_string::copy_printable_hhhh(CHARSET_INFO *to_cs, + CHARSET_INFO *from_cs, + const char *from, + uint32 from_length) +{ + uint errors; + uint one_escaped_char_length= MY_CS_PRINTABLE_CHAR_LENGTH * to_cs->mbminlen; + uint one_char_length= MY_MAX(one_escaped_char_length, to_cs->mbmaxlen); + uint32 bytes_needed= (uint32) from_length * one_char_length; + if (alloc(bytes_needed)) + return true; + str_length= my_convert_using_func(Ptr, Alloced_length, to_cs, + my_wc_to_printable_generic, + from, from_length, + from_cs, + from_cs->cset->mb_wc, + &errors); + return false; +} + + /* Compare strings according to collation, without end space. diff --git a/sql/sql_string.h b/sql/sql_string.h index 317adc13909..36a5a5d5e84 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -524,6 +524,15 @@ public: bool copy(const char *s, size_t arg_length); // Allocate new string bool copy_or_move(const char *s,size_t arg_length); + /** + Convert a string to a printable format. + All non-convertable and control characters are replaced to 5-character + sequences '\hhhh'. + */ + bool copy_printable_hhhh(CHARSET_INFO *to_cs, + CHARSET_INFO *from_cs, + const char *from, uint32 from_length); + bool append_ulonglong(ulonglong val); bool append_longlong(longlong val); diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/transaction_rollback_delete_delete.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/transaction_rollback_delete_delete.result index 9082032ff6e..8c61d9e865a 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/transaction_rollback_delete_delete.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/transaction_rollback_delete_delete.result @@ -28,7 +28,7 @@ MATCH(body) AGAINST("groonga"); id title body DELETE FROM diaries WHERE id = 1; Warnings: -Warning 1026 failed to get record ID for deleting from groonga: key=<> +Warning 1026 failed to get record ID for deleting from groonga: key=<\0001> SELECT * FROM diaries; id title body 2 groonga (1) starting groonga... diff --git a/strings/ctype.c b/strings/ctype.c index 9a89c6fe41d..fad100f638a 100644 --- a/strings/ctype.c +++ b/strings/ctype.c @@ -985,6 +985,100 @@ my_charset_is_ascii_based(CHARSET_INFO *cs) } +/** + Detect if a Unicode code point is printable. +*/ +static inline my_bool +my_is_printable(my_wc_t wc) +{ + /* + Blocks: + U+0000 .. U+001F control + U+0020 .. U+007E printable + U+007F .. U+009F control + U+00A0 .. U+00FF printable + U+0100 .. U+10FFFF As of Unicode-6.1.0, this range does not have any + characters of the "Cc" (Other, control) category. + Should be mostly safe to print. + Except for the surrogate halfs, + which are encoding components, not real characters. + */ + if (wc >= 0x20 && wc <= 0x7E) /* Quickly detect ASCII printable */ + return TRUE; + if (wc <= 0x9F) /* The rest of U+0000..U+009F are control characters */ + { + /* NL, CR, TAB are Ok */ + return (wc == '\r' || wc == '\n' || wc == '\t'); + } + /* + Surrogate halfs (when alone) print badly in terminals: + SELECT _ucs2 0xD800; + Let's escape them as well. + */ + if (wc >= 0xD800 && wc <= 0xDFFF) + return FALSE; + return TRUE; +} + + +static uint to_printable_8bit(uchar *dst, my_wc_t wc) +{ + /* + This function is used only in context of error messages for now. + All non-BMP characters are currently replaced to question marks + when a message is put into diagnostics area. + */ + DBUG_ASSERT(wc < 0x10000); + *dst++= '\\'; + *dst++= _dig_vec_upper[(wc >> 12) & 0x0F]; + *dst++= _dig_vec_upper[(wc >> 8) & 0x0F]; + *dst++= _dig_vec_upper[(wc >> 4) & 0x0F]; + *dst++= _dig_vec_upper[wc & 0x0F]; + return MY_CS_PRINTABLE_CHAR_LENGTH; +} + + +/** + Encode an Unicode character "wc" into a printable string. + This function is suitable for any character set, including + ASCII-incompatible multi-byte character sets, e.g. ucs2, utf16, utf32. +*/ +int +my_wc_to_printable_generic(CHARSET_INFO *cs, my_wc_t wc, + uchar *str, uchar *end) +{ + uchar *str0; + uint i, length; + uchar tmp[MY_CS_PRINTABLE_CHAR_LENGTH]; + + if (my_is_printable(wc)) + { + int mblen= cs->cset->wc_mb(cs, wc, str, end); + if (mblen > 0) + return mblen; + } + + if (str + MY_CS_PRINTABLE_CHAR_LENGTH * cs->mbminlen > end) + return MY_CS_TOOSMALLN(MY_CS_PRINTABLE_CHAR_LENGTH * cs->mbminlen); + + if ((cs->state & MY_CS_NONASCII) == 0) + return to_printable_8bit(str, wc); + + length= to_printable_8bit(tmp, wc); + str0= str; + for (i= 0; i < length; i++) + { + if (cs->cset->wc_mb(cs, tmp[i], str, end) != (int) cs->mbminlen) + { + DBUG_ASSERT(0); + return MY_CS_ILSEQ; + } + str+= cs->mbminlen; + } + return str - str0; +} + + /* Convert a string between two character sets. 'to' must be large enough to store (form_length * to_cs->mbmaxlen) bytes.