From beb1e230ddbf6b8a6bc22666df3147243a1a43e6 Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Mon, 16 Jan 2023 17:45:06 +0200 Subject: [PATCH 1/6] MDEV-30419 Fix unhandled exception thrown from wsrep-lib Updated wsrep-lib to version in which server_state wait_until_state() and sst_received() were changed to report errors via return codes instead of throwing exceptions. Added error handling accordingly. Tested manually that failure in sst_received() which was caused by server misconfiguration (unknown configuration variable in server configuration) does not cause crash due to uncaught exception. --- sql/wsrep_mysqld.cc | 22 ++++++++++++++++++---- sql/wsrep_sst.cc | 11 ++++++++--- wsrep-lib | 2 +- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 53212e2433c..a95cefc6f0f 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -915,13 +915,19 @@ void wsrep_init_startup (bool sst_first) With mysqldump SST (!sst_first) wait until the server reaches joiner state and procedd to accepting connections. */ + int err= 0; if (sst_first) { - server_state.wait_until_state(Wsrep_server_state::s_initializing); + err= server_state.wait_until_state(Wsrep_server_state::s_initializing); } else { - server_state.wait_until_state(Wsrep_server_state::s_joiner); + err= server_state.wait_until_state(Wsrep_server_state::s_joiner); + } + if (err) + { + WSREP_ERROR("Wsrep startup was interrupted"); + unireg_abort(1); } } @@ -1016,7 +1022,11 @@ void wsrep_stop_replication(THD *thd) { WSREP_DEBUG("Disconnect provider"); Wsrep_server_state::instance().disconnect(); - Wsrep_server_state::instance().wait_until_state(Wsrep_server_state::s_disconnected); + if (Wsrep_server_state::instance().wait_until_state( + Wsrep_server_state::s_disconnected)) + { + WSREP_WARN("Wsrep interrupted while waiting for disconnected state"); + } } /* my connection, should not terminate with wsrep_close_client_connection(), @@ -1038,7 +1048,11 @@ void wsrep_shutdown_replication() { WSREP_DEBUG("Disconnect provider"); Wsrep_server_state::instance().disconnect(); - Wsrep_server_state::instance().wait_until_state(Wsrep_server_state::s_disconnected); + if (Wsrep_server_state::instance().wait_until_state( + Wsrep_server_state::s_disconnected)) + { + WSREP_WARN("Wsrep interrupted while waiting for disconnected state"); + } } wsrep_close_client_connections(TRUE); diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index ddd5b5945b4..1fb1d6890e2 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -336,9 +336,14 @@ static bool wsrep_sst_complete (THD* thd, if ((state == Wsrep_server_state::s_joiner || state == Wsrep_server_state::s_initialized)) { - Wsrep_server_state::instance().sst_received(client_service, - rcode); - WSREP_INFO("SST succeeded for position %s", start_pos_buf); + if (Wsrep_server_state::instance().sst_received(client_service, rcode)) + { + failed= true; + } + else + { + WSREP_INFO("SST succeeded for position %s", start_pos_buf); + } } else { diff --git a/wsrep-lib b/wsrep-lib index f8ff2cfdd4c..275a0af8c5b 160000 --- a/wsrep-lib +++ b/wsrep-lib @@ -1 +1 @@ -Subproject commit f8ff2cfdd4c6424ffd96fc53bcc0f2e1d9ffe137 +Subproject commit 275a0af8c5b92f0ee33cfe9e23f3db5f59b56e9d From a27b8b26832340a293ee26be5a6c9821b1df1b5e Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 28 Oct 2022 13:43:51 +0400 Subject: [PATCH 2/6] MDEV-27653 long uniques don't work with unicode collations --- mysql-test/main/ctype_utf8.result | 175 ++++++++++++++++++ mysql-test/main/ctype_utf8.test | 158 ++++++++++++++++ mysql-test/main/type_timestamp.result | 22 +++ mysql-test/main/type_timestamp.test | 21 +++ .../mdev27653_100422_myisam_text.MYD | Bin 0 -> 40 bytes .../mdev27653_100422_myisam_text.MYI | Bin 0 -> 2048 bytes .../mdev27653_100422_myisam_text.frm | Bin 0 -> 970 bytes sql/field.cc | 75 ++++---- sql/field.h | 14 +- sql/ha_partition.cc | 17 +- sql/handler.cc | 32 ++++ sql/handler.h | 1 + sql/item.h | 13 ++ sql/item_func.cc | 20 +- sql/item_func.h | 12 ++ sql/item_strfunc.cc | 12 ++ sql/item_strfunc.h | 1 + sql/sql_admin.cc | 35 +++- sql/sql_alter.cc | 2 + sql/sql_class.cc | 13 ++ sql/sql_class.h | 25 +++ sql/sql_parse.cc | 4 +- sql/sql_table.cc | 19 +- sql/sql_table.h | 4 +- sql/sql_type.h | 26 +++ sql/table.cc | 17 +- sql/table.h | 16 ++ 27 files changed, 668 insertions(+), 66 deletions(-) create mode 100644 mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD create mode 100644 mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI create mode 100644 mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm diff --git a/mysql-test/main/ctype_utf8.result b/mysql-test/main/ctype_utf8.result index 5356b8d04c8..955530c6b14 100644 --- a/mysql-test/main/ctype_utf8.result +++ b/mysql-test/main/ctype_utf8.result @@ -11379,3 +11379,178 @@ a # # End of 10.3 tests # +# +# Start of 10.4 tests +# +# +# MDEV-27653 long uniques don't work with unicode collations +# +SET NAMES utf8mb3; +CREATE TABLE t1 ( +a CHAR(30) COLLATE utf8mb3_general_ci, +UNIQUE KEY(a) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` char(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 ( +a CHAR(30) COLLATE utf8mb3_general_ci, +UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` char(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`(10)) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 ( +a VARCHAR(30) COLLATE utf8mb3_general_ci, +UNIQUE KEY(a) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 ( +a VARCHAR(30) COLLATE utf8mb3_general_ci, +UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`(10)) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 (a TEXT COLLATE utf8mb3_general_ci UNIQUE); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 ( +a LONGTEXT COLLATE utf8mb3_general_ci, +UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` longtext CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`(10)) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +ä 2 +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check error Upgrade required. Please do "REPAIR TABLE `t1`" or dump/reload to fix it! +INSERT INTO t1 VALUES ('A'); +ERROR 23000: Duplicate entry 'A' for key 'a' +INSERT INTO t1 VALUES ('Ä'); +ERROR 23000: Duplicate entry 'Ä' for key 'a' +INSERT INTO t1 VALUES ('Ấ'); +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +ä 2 +Ấ 3 +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check error Upgrade required. Please do "REPAIR TABLE `t1`" or dump/reload to fix it! +ALTER TABLE t1 FORCE; +ERROR 23000: Duplicate entry 'ä' for key 'a' +DELETE FROM t1 WHERE OCTET_LENGTH(a)>1; +ALTER TABLE t1 FORCE; +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +DROP TABLE t1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +ä 2 +ALTER IGNORE TABLE t1 FORCE; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +DROP TABLE t1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +ä 2 +REPAIR TABLE t1; +Table Op Msg_type Msg_text +test.t1 repair Warning Number of rows changed from 2 to 1 +test.t1 repair status OK +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +DROP TABLE t1; +# +# End of 10.4 tests +# diff --git a/mysql-test/main/ctype_utf8.test b/mysql-test/main/ctype_utf8.test index cc61c2ae0fe..027881bf0eb 100644 --- a/mysql-test/main/ctype_utf8.test +++ b/mysql-test/main/ctype_utf8.test @@ -2310,3 +2310,161 @@ VALUES (_latin1 0xDF) UNION VALUES(_utf8'a' COLLATE utf8_bin); --echo # --echo # End of 10.3 tests --echo # + + +--echo # +--echo # Start of 10.4 tests +--echo # + +--echo # +--echo # MDEV-27653 long uniques don't work with unicode collations +--echo # + +SET NAMES utf8mb3; + +# CHAR + +CREATE TABLE t1 ( + a CHAR(30) COLLATE utf8mb3_general_ci, + UNIQUE KEY(a) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + a CHAR(30) COLLATE utf8mb3_general_ci, + UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + + +# VARCHAR + +CREATE TABLE t1 ( + a VARCHAR(30) COLLATE utf8mb3_general_ci, + UNIQUE KEY(a) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + a VARCHAR(30) COLLATE utf8mb3_general_ci, + UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + + +# TEXT + +CREATE TABLE t1 (a TEXT COLLATE utf8mb3_general_ci UNIQUE); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + a LONGTEXT COLLATE utf8mb3_general_ci, + UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + + +# Testing upgrade: +# Prior to MDEV-27653, the UNIQUE HASH function errorneously +# took into account string octet length. +# Old tables should still open and work, but with wrong results. + +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI; +SHOW CREATE TABLE t1; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +CHECK TABLE t1; + +# There is already a one byte value 'a' in the table +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('A'); + +# There is already a two-byte value 'ä' in the table +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('Ä'); + +# There were no three-byte values in the table so far. +# The below value violates UNIQUE, but it gets inserted. +# This is wrong but expected for a pre-MDEV-27653 table. +INSERT INTO t1 VALUES ('Ấ'); +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +CHECK TABLE t1; + +# ALTER FORCE fails: it tries to rebuild the table +# with a correct UNIQUE HASH function, but there are duplicates! +--error ER_DUP_ENTRY +ALTER TABLE t1 FORCE; + +# Let's remove all duplicate values, so only the one-byte 'a' stays. +# ALTER..FORCE should work after that. +DELETE FROM t1 WHERE OCTET_LENGTH(a)>1; +ALTER TABLE t1 FORCE; + +# Make sure that 'a' and 'ä' cannot co-exists any more, +# because the table was recreated with a correct UNIQUE HASH function. +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +DROP TABLE t1; + +# +# Testing an old table with ALTER IGNORE. +# The table is expected to rebuild with a new hash function, +# duplicates go away. +# +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI; +SHOW CREATE TABLE t1; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +ALTER IGNORE TABLE t1 FORCE; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +DROP TABLE t1; + +# +# Testing an old table with REPAIR. +# The table is expected to rebuild with a new hash function, +# duplicates go away. +# +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI; +SHOW CREATE TABLE t1; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +REPAIR TABLE t1; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +DROP TABLE t1; + +--echo # +--echo # End of 10.4 tests +--echo # diff --git a/mysql-test/main/type_timestamp.result b/mysql-test/main/type_timestamp.result index 306cd77ad25..58cb12ae267 100644 --- a/mysql-test/main/type_timestamp.result +++ b/mysql-test/main/type_timestamp.result @@ -1320,5 +1320,27 @@ CASE WHEN a THEN DEFAULT(a) END DROP TABLE t1; SET timestamp=DEFAULT; # +# MDEV-27653 long uniques don't work with unicode collations +# +CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH); +SET time_zone='+00:00'; +INSERT INTO t1 VALUES ('2001-01-01 10:20:30'); +SET time_zone='+01:00'; +INSERT INTO t1 SELECT MAX(a) FROM t1; +ERROR 23000: Duplicate entry '2001-01-01 11:20:30' for key 'a' +SELECT * FROM t1; +a +2001-01-01 11:20:30 +DROP TABLE t1; +CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH); +SET time_zone='+00:00'; +INSERT INTO t1 VALUES ('2001-01-01 10:20:30'); +SET time_zone='+01:00'; +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +DROP TABLE t1; +SET time_zone=DEFAULT; +# # End of 10.4 tests # diff --git a/mysql-test/main/type_timestamp.test b/mysql-test/main/type_timestamp.test index 485da666661..f12cc2a4bc3 100644 --- a/mysql-test/main/type_timestamp.test +++ b/mysql-test/main/type_timestamp.test @@ -878,6 +878,27 @@ DROP TABLE t1; SET timestamp=DEFAULT; +--echo # +--echo # MDEV-27653 long uniques don't work with unicode collations +--echo # + +CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH); +SET time_zone='+00:00'; +INSERT INTO t1 VALUES ('2001-01-01 10:20:30'); +SET time_zone='+01:00'; +--error ER_DUP_ENTRY +INSERT INTO t1 SELECT MAX(a) FROM t1; +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH); +SET time_zone='+00:00'; +INSERT INTO t1 VALUES ('2001-01-01 10:20:30'); +SET time_zone='+01:00'; +CHECK TABLE t1; +DROP TABLE t1; +SET time_zone=DEFAULT; + --echo # --echo # End of 10.4 tests --echo # diff --git a/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD new file mode 100644 index 0000000000000000000000000000000000000000..5f1ad998c00b384520a17268b3070617b3ff06a8 GIT binary patch literal 40 fcmZQ(VC81`!^n`xfB?)4Y+MX~m>3Q(L5KkWQVj#@ literal 0 HcmV?d00001 diff --git a/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI new file mode 100644 index 0000000000000000000000000000000000000000..7a748abbae59ebbba31b279a90b83968938eff3d GIT binary patch literal 2048 zcmezOkDZZ;nK6W6149bK5e7yEAmRX$3=Ee4K_n0`K`1!M^dAZ!A{;OY4HyF_&H3IG zC}#$hhnma6fNCBCLvr-itxz>!mmu+R8qL7Kf=3QTiW|rSVJ4tgd4RMCm<{we8yndB zj6jnlfMO_sk(I#%D9#Up-1cCn2QYxV4l)S@z%FHB-~!VSp;2-)1O|NwC_oY{r>+Xb YsSvK*r#E0OC~u5{(GVC7fngc~0HT5)jQ{`u literal 0 HcmV?d00001 diff --git a/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm new file mode 100644 index 0000000000000000000000000000000000000000..5a4cec723244cd7562045e933b3164dd5568dcea GIT binary patch literal 970 zcmeyz$jvFn5XHdAaEh6M;U}Xo0~|0(GjK96u!8s;Fu|XU3=D1^j0^%W0S1A@1y@VL z)(gI!arjBdi;}NT8<`lG%a|A#7=f6D!2zg{5oiWu1CT4w!ocu9@jqPeC}TK=0M__r zWALr?40iN|csPVH1elOUG63cVa3&IF5P}Co0D~9coll->hash_sort(cs, ptr, len, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + hasher->add(sort_charset(), ptr, pack_length()); } size_t @@ -8180,18 +8173,12 @@ bool Field_varstring::is_equal(const Column_definition &new_field) const } -void Field_varstring::hash(ulong *nr, ulong *nr2) +void Field_varstring::hash_not_null(Hasher *hasher) { - if (is_null()) - { - *nr^= (*nr << 1) | 1; - } - else - { - uint len= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); - CHARSET_INFO *cs= charset(); - cs->coll->hash_sort(cs, ptr + length_bytes, len, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + uint len= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); + hasher->add(charset(), ptr + length_bytes, len); } @@ -8553,6 +8540,17 @@ oom_error: } +void Field_blob::hash_not_null(Hasher *hasher) +{ + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + char *blob; + memcpy(&blob, ptr + packlength, sizeof(char*)); + if (blob) + hasher->add(Field_blob::charset(), blob, get_length(ptr)); +} + + double Field_blob::val_real(void) { DBUG_ASSERT(marked_for_read()); @@ -9901,20 +9899,27 @@ Field_bit::Field_bit(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, } -void Field_bit::hash(ulong *nr, ulong *nr2) +/* + This method always calculates hash over 8 bytes. + This is different from how the HEAP engine calculate hash: + HEAP takes into account the actual octet size, so say for BIT(18) + it calculates hash over three bytes only: + - the incomplete byte with bits 16..17 + - the two full bytes with bits 0..15 + See hp_rec_hashnr(), hp_hashnr() for details. + + The HEAP way is more efficient, especially for short lengths. + Let's consider fixing Field_bit eventually to do it in the HEAP way, + with proper measures to upgrade partitioned tables easy. +*/ +void Field_bit::hash_not_null(Hasher *hasher) { - if (is_null()) - { - *nr^= (*nr << 1) | 1; - } - else - { - CHARSET_INFO *cs= &my_charset_bin; - longlong value= Field_bit::val_int(); - uchar tmp[8]; - mi_int8store(tmp,value); - cs->coll->hash_sort(cs, tmp, 8, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + longlong value= Field_bit::val_int(); + uchar tmp[8]; + mi_int8store(tmp,value); + hasher->add(&my_charset_bin, tmp, 8); } diff --git a/sql/field.h b/sql/field.h index d4124ca23a9..6388fa2e08e 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1628,7 +1628,14 @@ public: key_map get_possible_keys(); /* Hash value */ - virtual void hash(ulong *nr, ulong *nr2); + void hash(Hasher *hasher) + { + if (is_null()) + hasher->add_null(); + else + hash_not_null(hasher); + } + virtual void hash_not_null(Hasher *hasher); /** Get the upper limit of the MySQL integral and floating-point type. @@ -3744,7 +3751,7 @@ public: uchar *new_ptr, uint32 length, uchar *new_null_ptr, uint new_null_bit); bool is_equal(const Column_definition &new_field) const; - void hash(ulong *nr, ulong *nr2); + void hash_not_null(Hasher *hasher); uint length_size() const { return length_bytes; } void print_key_value(String *out, uint32 length); private: @@ -3951,6 +3958,7 @@ public: bool make_empty_rec_store_default_value(THD *thd, Item *item); int store(const char *to, size_t length, CHARSET_INFO *charset); using Field_str::store; + void hash_not_null(Hasher *hasher); double val_real(void); longlong val_int(void); String *val_str(String*,String *); @@ -4576,7 +4584,7 @@ public: if (bit_ptr) bit_ptr= ADD_TO_PTR(bit_ptr, ptr_diff, uchar*); } - void hash(ulong *nr, ulong *nr2); + void hash_not_null(Hasher *hasher); SEL_ARG *get_mm_leaf(RANGE_OPT_PARAM *param, KEY_PART *key_part, const Item_bool_func *cond, diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 06ae329ee3a..ca02bf16d5e 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -9850,8 +9850,7 @@ uint8 ha_partition::table_cache_type() uint32 ha_partition::calculate_key_hash_value(Field **field_array) { - ulong nr1= 1; - ulong nr2= 4; + Hasher hasher; bool use_51_hash; use_51_hash= MY_TEST((*field_array)->table->part_info->key_algorithm == partition_info::KEY_ALGORITHM_51); @@ -9878,13 +9877,12 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) { if (field->is_null()) { - nr1^= (nr1 << 1) | 1; + hasher.add_null(); continue; } /* Force this to my_hash_sort_bin, which was used in 5.1! */ uint len= field->pack_length(); - my_charset_bin.coll->hash_sort(&my_charset_bin, field->ptr, len, - &nr1, &nr2); + hasher.add(&my_charset_bin, field->ptr, len); /* Done with this field, continue with next one. */ continue; } @@ -9902,13 +9900,12 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) { if (field->is_null()) { - nr1^= (nr1 << 1) | 1; + hasher.add_null(); continue; } /* Force this to my_hash_sort_bin, which was used in 5.1! */ uint len= field->pack_length(); - my_charset_latin1.coll->hash_sort(&my_charset_latin1, field->ptr, - len, &nr1, &nr2); + hasher.add(&my_charset_latin1, field->ptr, len); continue; } /* New types in mysql-5.6. */ @@ -9935,9 +9932,9 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) } /* fall through, use collation based hashing. */ } - field->hash(&nr1, &nr2); + field->hash(&hasher); } while (*(++field_array)); - return (uint32) nr1; + return (uint32) hasher.finalize(); } diff --git a/sql/handler.cc b/sql/handler.cc index 20bf32f3cdc..42bfec6652f 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4145,6 +4145,35 @@ int handler::check_collation_compatibility() } +int handler::check_long_hash_compatibility() const +{ + if (!table->s->old_long_hash_function()) + return 0; + KEY *key= table->key_info; + KEY *key_end= key + table->s->keys; + for ( ; key < key_end; key++) + { + if (key->algorithm == HA_KEY_ALG_LONG_HASH) + { + /* + The old (pre-MDEV-27653) hash function was wrong. + So the long hash unique constraint can have some + duplicate records. REPAIR TABLE can't fix this, + it will fail on a duplicate key error. + Only "ALTER IGNORE TABLE .. FORCE" can fix this. + So we need to return HA_ADMIN_NEEDS_ALTER here, + (not HA_ADMIN_NEEDS_UPGRADE which is used elsewhere), + to properly send the error message text corresponding + to ER_TABLE_NEEDS_REBUILD (rather than to ER_TABLE_NEEDS_UPGRADE) + to the user. + */ + return HA_ADMIN_NEEDS_ALTER; + } + } + return 0; +} + + int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt) { int error; @@ -4182,6 +4211,9 @@ int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt) if (unlikely((error= check_collation_compatibility()))) return error; + + if (unlikely((error= check_long_hash_compatibility()))) + return error; return check_for_upgrade(check_opt); } diff --git a/sql/handler.h b/sql/handler.h index 3f0fc5c897f..f7a2838b3d9 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -3271,6 +3271,7 @@ public: } int check_collation_compatibility(); + int check_long_hash_compatibility() const; int ha_check_for_upgrade(HA_CHECK_OPT *check_opt); /** to be actually called to get 'check()' functionality*/ int ha_check(THD *thd, HA_CHECK_OPT *check_opt); diff --git a/sql/item.h b/sql/item.h index ac4a42107cb..4fbb7324570 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1271,6 +1271,12 @@ public: */ inline ulonglong val_uint() { return (ulonglong) val_int(); } + virtual bool hash_not_null(Hasher *hasher) + { + DBUG_ASSERT(0); + return true; + } + /* Return string representation of this item object. @@ -3460,6 +3466,13 @@ public: { return Sql_mode_dependency(0, field->value_depends_on_sql_mode()); } + bool hash_not_null(Hasher *hasher) + { + if (field->is_null()) + return true; + field->hash_not_null(hasher); + return false; + } longlong val_int_endpoint(bool left_endp, bool *incl_endp); bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate); bool get_date_result(THD *thd, MYSQL_TIME *ltime,date_mode_t fuzzydate); diff --git a/sql/item_func.cc b/sql/item_func.cc index 9c29280970b..d275dbaa50d 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -1765,7 +1765,7 @@ static void calc_hash_for_unique(ulong &nr1, ulong &nr2, String *str) cs->coll->hash_sort(cs, (uchar *)str->ptr(), str->length(), &nr1, &nr2); } -longlong Item_func_hash::val_int() +longlong Item_func_hash_mariadb_100403::val_int() { DBUG_EXECUTE_IF("same_long_unique_hash", return 9;); unsigned_flag= true; @@ -1786,6 +1786,24 @@ longlong Item_func_hash::val_int() } +longlong Item_func_hash::val_int() +{ + DBUG_EXECUTE_IF("same_long_unique_hash", return 9;); + unsigned_flag= true; + Hasher hasher; + for(uint i= 0;ihash_not_null(&hasher)) + { + null_value= 1; + return 0; + } + } + null_value= 0; + return (longlong) hasher.finalize(); +} + + bool Item_func_hash::fix_length_and_dec() { decimals= 0; diff --git a/sql/item_func.h b/sql/item_func.h index 7457abd9a39..ef36d5204d2 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1078,6 +1078,18 @@ public: const char *func_name() const { return ""; } }; +class Item_func_hash_mariadb_100403: public Item_func_hash +{ +public: + Item_func_hash_mariadb_100403(THD *thd, List &item) + :Item_func_hash(thd, item) + {} + longlong val_int(); + Item *get_copy(THD *thd) + { return get_item_copy(thd, this); } + const char *func_name() const { return ""; } +}; + class Item_longlong_func: public Item_int_func { public: diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 5562e6d3e62..a57fbd7bebc 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -1630,6 +1630,18 @@ bool Item_func_ucase::fix_length_and_dec() } +bool Item_func_left::hash_not_null(Hasher *hasher) +{ + StringBuffer buf; + String *str= val_str(&buf); + DBUG_ASSERT((str == NULL) == null_value); + if (!str) + return true; + hasher->add(collation.collation, str->ptr(), str->length()); + return false; +} + + String *Item_func_left::val_str(String *str) { DBUG_ASSERT(fixed == 1); diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 82b30369734..ee2c59c973f 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -461,6 +461,7 @@ class Item_func_left :public Item_str_func String tmp_value; public: Item_func_left(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} + bool hash_not_null(Hasher *hasher); String *val_str(String *); bool fix_length_and_dec(); const char *func_name() const { return "left"; } diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 213d77f8237..906458b7b3c 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -35,7 +35,8 @@ #include "wsrep_mysqld.h" /* Prepare, run and cleanup for mysql_recreate_table() */ -static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list) +static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list, + Recreate_info *recreate_info) { bool result_code; DBUG_ENTER("admin_recreate_table"); @@ -56,7 +57,7 @@ static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list) DEBUG_SYNC(thd, "ha_admin_try_alter"); tmp_disable_binlog(thd); // binlogging is done by caller if wanted result_code= (thd->open_temporary_tables(table_list) || - mysql_recreate_table(thd, table_list, false)); + mysql_recreate_table(thd, table_list, recreate_info, false)); reenable_binlog(thd); /* mysql_recreate_table() can push OK or ERROR. @@ -516,6 +517,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, bool open_error; bool collect_eis= FALSE; bool open_for_modify= org_open_for_modify; + Recreate_info recreate_info; DBUG_PRINT("admin", ("table: '%s'.'%s'", db, table->table_name.str)); DEBUG_SYNC(thd, "admin_command_kill_before_modify"); @@ -776,7 +778,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, { /* We use extra_open_options to be able to open crashed tables */ thd->open_options|= extra_open_options; - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info) ? + HA_ADMIN_FAILED : HA_ADMIN_OK; thd->open_options&= ~extra_open_options; goto send_result; } @@ -947,12 +950,31 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, repair was not implemented and we need to upgrade the table to a new version so we recreate the table with ALTER TABLE */ - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info); } send_result: lex->cleanup_after_one_table_open(); thd->clear_error(); // these errors shouldn't get client + + if (recreate_info.records_duplicate()) + { + protocol->prepare_for_resend(); + protocol->store(table_name, system_charset_info); + protocol->store((char*) operator_name, system_charset_info); + protocol->store(warning_level_names[Sql_condition::WARN_LEVEL_WARN].str, + warning_level_names[Sql_condition::WARN_LEVEL_WARN].length, + system_charset_info); + char buf[80]; + size_t length= my_snprintf(buf, sizeof(buf), + "Number of rows changed from %u to %u", + (uint) recreate_info.records_processed(), + (uint) recreate_info.records_copied()); + protocol->store(buf, length, system_charset_info); + if (protocol->write()) + goto err; + } + { Diagnostics_area::Sql_condition_iterator it= thd->get_stmt_da()->sql_conditions(); @@ -1063,7 +1085,7 @@ send_result_message: table->next_local= table->next_global= 0; tmp_disable_binlog(thd); // binlogging is done by caller if wanted - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info); reenable_binlog(thd); trans_commit_stmt(thd); trans_commit(thd); @@ -1409,6 +1431,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd) LEX *m_lex= thd->lex; TABLE_LIST *first_table= m_lex->first_select_lex()->table_list.first; bool res= TRUE; + Recreate_info recreate_info; DBUG_ENTER("Sql_cmd_optimize_table::execute"); if (check_table_access(thd, SELECT_ACL | INSERT_ACL, first_table, @@ -1417,7 +1440,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd) WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table); res= (specialflag & SPECIAL_NO_NEW_FUNC) ? - mysql_recreate_table(thd, first_table, true) : + mysql_recreate_table(thd, first_table, &recreate_info, true) : mysql_admin_table(thd, first_table, &m_lex->check_opt, "optimize", TL_WRITE, 1, 0, 0, 0, &handler::ha_optimize, 0, true); diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc index 67c6a081880..cf44ed67120 100644 --- a/sql/sql_alter.cc +++ b/sql/sql_alter.cc @@ -527,9 +527,11 @@ bool Sql_cmd_alter_table::execute(THD *thd) thd->work_part_info= 0; #endif + Recreate_info recreate_info; result= mysql_alter_table(thd, &select_lex->db, &lex->name, &create_info, first_table, + &recreate_info, &alter_info, select_lex->order_list.elements, select_lex->order_list.first, diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 464d64db73b..300ed902b38 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -7939,3 +7939,16 @@ bool THD::timestamp_to_TIME(MYSQL_TIME *ltime, my_time_t ts, } return 0; } + + +void THD::my_ok_with_recreate_info(const Recreate_info &info, + ulong warn_count) +{ + char buf[80]; + my_snprintf(buf, sizeof(buf), + ER_THD(this, ER_INSERT_INFO), + (ulong) info.records_processed(), + (ulong) info.records_duplicate(), + warn_count); + my_ok(this, info.records_processed(), 0L, buf); +} diff --git a/sql/sql_class.h b/sql/sql_class.h index 136380bbd87..67018c0299a 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -232,6 +232,29 @@ public: }; +class Recreate_info +{ + ha_rows m_records_copied; + ha_rows m_records_duplicate; +public: + Recreate_info() + :m_records_copied(0), + m_records_duplicate(0) + { } + Recreate_info(ha_rows records_copied, + ha_rows records_duplicate) + :m_records_copied(records_copied), + m_records_duplicate(records_duplicate) + { } + ha_rows records_copied() const { return m_records_copied; } + ha_rows records_duplicate() const { return m_records_duplicate; } + ha_rows records_processed() const + { + return m_records_copied + m_records_duplicate; + } +}; + + #define TC_HEURISTIC_RECOVER_COMMIT 1 #define TC_HEURISTIC_RECOVER_ROLLBACK 2 extern ulong tc_heuristic_recover; @@ -3943,6 +3966,8 @@ public: inline bool vio_ok() const { return TRUE; } inline bool is_connected() { return TRUE; } #endif + + void my_ok_with_recreate_info(const Recreate_info &info, ulong warn_count); /** Mark the current error as fatal. Warning: this does not set any error, it sets a property of the error, so must be diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index f8974bb9cc0..bb0b3ebb4da 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4240,8 +4240,10 @@ mysql_execute_command(THD *thd) create_info.row_type= ROW_TYPE_NOT_USED; create_info.default_table_charset= thd->variables.collation_database; + Recreate_info recreate_info; res= mysql_alter_table(thd, &first_table->db, &first_table->table_name, - &create_info, first_table, &alter_info, + &create_info, first_table, + &recreate_info, &alter_info, 0, (ORDER*) 0, 0); break; } diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 725af4adb4e..9f13dcde40f 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -9619,6 +9619,7 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name, HA_CREATE_INFO *create_info, TABLE_LIST *table_list, + Recreate_info *recreate_info, Alter_info *alter_info, uint order_num, ORDER *order, bool ignore) { @@ -10687,11 +10688,10 @@ end_inplace: } end_temporary: - my_snprintf(alter_ctx.tmp_buff, sizeof(alter_ctx.tmp_buff), - ER_THD(thd, ER_INSERT_INFO), - (ulong) (copied + deleted), (ulong) deleted, - (ulong) thd->get_stmt_da()->current_statement_warn_count()); - my_ok(thd, copied + deleted, 0L, alter_ctx.tmp_buff); + *recreate_info= Recreate_info(copied, deleted); + thd->my_ok_with_recreate_info(*recreate_info, + (ulong) thd->get_stmt_da()-> + current_statement_warn_count()); DEBUG_SYNC(thd, "alter_table_inplace_trans_commit"); DBUG_RETURN(false); @@ -11208,7 +11208,8 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, Like mysql_alter_table(). */ -bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy) +bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, + Recreate_info *recreate_info, bool table_copy) { HA_CREATE_INFO create_info; Alter_info alter_info; @@ -11233,8 +11234,10 @@ bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy) Alter_info::ALTER_TABLE_ALGORITHM_COPY); bool res= mysql_alter_table(thd, &null_clex_str, &null_clex_str, &create_info, - table_list, &alter_info, 0, - (ORDER *) 0, 0); + table_list, recreate_info, &alter_info, 0, + (ORDER *) 0, + // Ignore duplicate records on REPAIR + thd->lex->sql_command == SQLCOM_REPAIR); table_list->next_global= next_table; DBUG_RETURN(res); } diff --git a/sql/sql_table.h b/sql/sql_table.h index 62b61684286..e51b5ec0f0f 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -220,13 +220,15 @@ bool mysql_trans_commit_alter_copy_data(THD *thd); bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name, HA_CREATE_INFO *create_info, TABLE_LIST *table_list, + class Recreate_info *recreate_info, Alter_info *alter_info, uint order_num, ORDER *order, bool ignore); bool mysql_compare_tables(TABLE *table, Alter_info *alter_info, HA_CREATE_INFO *create_info, bool *metadata_equal); -bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy); +bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, + class Recreate_info *recreate_info, bool table_copy); bool mysql_create_like_table(THD *thd, TABLE_LIST *table, TABLE_LIST *src_table, Table_specification_st *create_info); diff --git a/sql/sql_type.h b/sql/sql_type.h index ff6853b4c88..73e7de79cd8 100644 --- a/sql/sql_type.h +++ b/sql/sql_type.h @@ -110,6 +110,32 @@ enum scalar_comparison_op }; +class Hasher +{ + ulong m_nr1; + ulong m_nr2; +public: + Hasher(): m_nr1(1), m_nr2(4) + { } + void add_null() + { + m_nr1^= (m_nr1 << 1) | 1; + } + void add(CHARSET_INFO *cs, const uchar *str, size_t length) + { + cs->coll->hash_sort(cs, str, length, &m_nr1, &m_nr2); + } + void add(CHARSET_INFO *cs, const char *str, size_t length) + { + add(cs, (const uchar *) str, length); + } + uint32 finalize() const + { + return (uint32) m_nr1; + } +}; + + /* A helper class to store column attributes that are inherited by columns (from the table level) when not specified explicitly. diff --git a/sql/table.cc b/sql/table.cc index a06834afd8b..2b5787d1511 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -1056,6 +1056,18 @@ static void mysql57_calculate_null_position(TABLE_SHARE *share, } } + +Item_func_hash *TABLE_SHARE::make_long_hash_func(THD *thd, + MEM_ROOT *mem_root, + List *field_list) + const +{ + if (old_long_hash_function()) + return new (mem_root) Item_func_hash_mariadb_100403(thd, *field_list); + return new (mem_root) Item_func_hash(thd, *field_list); +} + + /** Parse TABLE_SHARE::vcol_defs unpack_vcol_info_from_frm @@ -1267,7 +1279,10 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table, list_item= new (mem_root) Item_field(thd, keypart->field); field_list->push_back(list_item, mem_root); } - Item_func_hash *hash_item= new(mem_root)Item_func_hash(thd, *field_list); + + Item_func_hash *hash_item= table->s->make_long_hash_func(thd, mem_root, + field_list); + Virtual_column_info *v= new (mem_root) Virtual_column_info(); field->vcol_info= v; field->vcol_info->expr= hash_item; diff --git a/sql/table.h b/sql/table.h index 1a10566e080..799675ac775 100644 --- a/sql/table.h +++ b/sql/table.h @@ -52,6 +52,7 @@ class Item; /* Needed by ORDER */ typedef Item (*Item_ptr); class Item_subselect; class Item_field; +class Item_func_hash; class GRANT_TABLE; class st_select_lex_unit; class st_select_lex; @@ -1137,6 +1138,21 @@ struct TABLE_SHARE void free_frm_image(const uchar *frm); void set_overlapped_keys(); + + bool old_long_hash_function() const + { + return mysql_version < 100428 || + (mysql_version >= 100500 && mysql_version < 100519) || + (mysql_version >= 100600 && mysql_version < 100612) || + (mysql_version >= 100700 && mysql_version < 100708) || + (mysql_version >= 100800 && mysql_version < 100807) || + (mysql_version >= 100900 && mysql_version < 100905) || + (mysql_version >= 101000 && mysql_version < 101003) || + (mysql_version >= 101100 && mysql_version < 101102); + } + Item_func_hash *make_long_hash_func(THD *thd, + MEM_ROOT *mem_root, + List *field_list) const; }; /* not NULL, but cannot be dereferenced */ From 0253a2f48e775f851dabc6df40660448ba56fa4f Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Thu, 12 Jan 2023 13:41:49 +1100 Subject: [PATCH 3/6] MDEV-26541 Make UBSAN builds work with spider again. When built with ubsan and trying to load the spider plugin, the hidden visibility of mysqld compiling flag causes ha_spider.so to be missing the symbol ha_partition. This commit fixes that, as well as some memcpy null pointer issues when built with ubsan. Signed-off-by: Yuchen Pei --- sql/CMakeLists.txt | 9 + .../spider/bugfix/r/mdev_26541.result | 20 ++ .../spider/bugfix/t/mdev_26541.test | 40 ++++ storage/spider/spd_conn.cc | 19 +- storage/spider/spd_trx.cc | 204 ++++++++++-------- 5 files changed, 200 insertions(+), 92 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_26541.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_26541.test diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index e7c87698451..968cab85e52 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -197,6 +197,15 @@ FOREACH(se aria partition perfschema sql_sequence wsrep) ENDIF() ENDFOREACH() +IF(VISIBILITY_HIDDEN_FLAG AND TARGET partition AND WITH_UBSAN) + # the spider plugin needs some partition symbols from inside mysqld + # when built with ubsan, in which case we need to remove + # -fvisibility=hidden from partition + GET_TARGET_PROPERTY(f partition COMPILE_FLAGS) + STRING(REPLACE "${VISIBILITY_HIDDEN_FLAG}" "" f ${f}) + SET_TARGET_PROPERTIES(partition PROPERTIES COMPILE_FLAGS "${f}") +ENDIF() + IF(WIN32) SET(MYSQLD_SOURCE main.cc nt_servc.cc message.rc) TARGET_LINK_LIBRARIES(sql psapi) diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_26541.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_26541.result new file mode 100644 index 00000000000..b2edaff6918 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_26541.result @@ -0,0 +1,20 @@ +# +# MDEV-26541 Undefined symbol: _ZTI12ha_partition when attempting to use ha_spider.so in UBSAN builds +# +INSTALL PLUGIN spider SONAME 'ha_spider.so'; +DROP FUNCTION spider_flush_table_mon_cache; +DROP FUNCTION spider_copy_tables; +DROP FUNCTION spider_ping_table; +DROP FUNCTION spider_bg_direct_sql; +DROP FUNCTION spider_direct_sql; +UNINSTALL PLUGIN spider_alloc_mem; +UNINSTALL PLUGIN spider; +DROP TABLE IF EXISTS mysql.spider_xa; +DROP TABLE IF EXISTS mysql.spider_xa_member; +DROP TABLE IF EXISTS mysql.spider_xa_failed_log; +DROP TABLE IF EXISTS mysql.spider_tables; +DROP TABLE IF EXISTS mysql.spider_link_mon_servers; +DROP TABLE IF EXISTS mysql.spider_link_failed_log; +DROP TABLE IF EXISTS mysql.spider_table_position_for_recovery; +DROP TABLE IF EXISTS mysql.spider_table_sts; +DROP TABLE IF EXISTS mysql.spider_table_crd; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_26541.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_26541.test new file mode 100644 index 00000000000..ffd99390748 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_26541.test @@ -0,0 +1,40 @@ +--echo # +--echo # MDEV-26541 Undefined symbol: _ZTI12ha_partition when attempting to use ha_spider.so in UBSAN builds +--echo # + +if (`select not(count(*)) from information_schema.system_variables where variable_name='have_sanitizer' and global_value="UBSAN"`) +{ +--skip test needs to be run with UBSAN +} + +# init spider + +INSTALL PLUGIN spider SONAME 'ha_spider.so'; + +let $PLUGIN_NAME= spider_flush_table_mon_cache; +let $PLUGIN_EXIST= + `SELECT COUNT(*) FROM mysql.func WHERE name = '$PLUGIN_NAME'`; +while (!$PLUGIN_EXIST) +{ + let $PLUGIN_EXIST= + `SELECT COUNT(*) FROM mysql.func WHERE name = '$PLUGIN_NAME'`; +} + +# deinit spider + +DROP FUNCTION spider_flush_table_mon_cache; +DROP FUNCTION spider_copy_tables; +DROP FUNCTION spider_ping_table; +DROP FUNCTION spider_bg_direct_sql; +DROP FUNCTION spider_direct_sql; +UNINSTALL PLUGIN spider_alloc_mem; +UNINSTALL PLUGIN spider; +DROP TABLE IF EXISTS mysql.spider_xa; +DROP TABLE IF EXISTS mysql.spider_xa_member; +DROP TABLE IF EXISTS mysql.spider_xa_failed_log; +DROP TABLE IF EXISTS mysql.spider_tables; +DROP TABLE IF EXISTS mysql.spider_link_mon_servers; +DROP TABLE IF EXISTS mysql.spider_link_failed_log; +DROP TABLE IF EXISTS mysql.spider_table_position_for_recovery; +DROP TABLE IF EXISTS mysql.spider_table_sts; +DROP TABLE IF EXISTS mysql.spider_table_crd; diff --git a/storage/spider/spd_conn.cc b/storage/spider/spd_conn.cc index cef4ba04eb1..1faa65d0db5 100644 --- a/storage/spider/spd_conn.cc +++ b/storage/spider/spd_conn.cc @@ -518,18 +518,25 @@ SPIDER_CONN *spider_create_conn( conn->tgt_host = tmp_host; memcpy(conn->tgt_host, share->tgt_hosts[link_idx], share->tgt_hosts_lengths[link_idx]); + conn->tgt_username_length = share->tgt_usernames_lengths[link_idx]; conn->tgt_username = tmp_username; - memcpy(conn->tgt_username, share->tgt_usernames[link_idx], - share->tgt_usernames_lengths[link_idx]); + if (conn->tgt_username_length) + memcpy(conn->tgt_username, share->tgt_usernames[link_idx], + share->tgt_usernames_lengths[link_idx]); + conn->tgt_password_length = share->tgt_passwords_lengths[link_idx]; conn->tgt_password = tmp_password; - memcpy(conn->tgt_password, share->tgt_passwords[link_idx], - share->tgt_passwords_lengths[link_idx]); + if (conn->tgt_password_length) + memcpy(conn->tgt_password, share->tgt_passwords[link_idx], + share->tgt_passwords_lengths[link_idx]); + conn->tgt_socket_length = share->tgt_sockets_lengths[link_idx]; conn->tgt_socket = tmp_socket; - memcpy(conn->tgt_socket, share->tgt_sockets[link_idx], - share->tgt_sockets_lengths[link_idx]); + if (conn->tgt_socket_length) + memcpy(conn->tgt_socket, share->tgt_sockets[link_idx], + share->tgt_sockets_lengths[link_idx]); + conn->tgt_wrapper_length = share->tgt_wrappers_lengths[link_idx]; conn->tgt_wrapper = tmp_wrapper; memcpy(conn->tgt_wrapper, share->tgt_wrappers[link_idx], diff --git a/storage/spider/spd_trx.cc b/storage/spider/spd_trx.cc index 4f8296f1d01..b0b1e9c36bd 100644 --- a/storage/spider/spd_trx.cc +++ b/storage/spider/spd_trx.cc @@ -682,112 +682,144 @@ int spider_create_trx_alter_table( alter_table->tmp_tgt_default_groups_lengths = tmp_tgt_default_groups_lengths; alter_table->tmp_static_link_ids_lengths = tmp_static_link_ids_lengths; + size_t len; for(roop_count = 0; roop_count < (int) share->all_link_count; roop_count++) { - tmp_server_names[roop_count] = tmp_server_names_char; - memcpy(tmp_server_names_char, - share_alter->tmp_server_names[roop_count], - sizeof(char) * share_alter->tmp_server_names_lengths[roop_count]); - tmp_server_names_char += - share_alter->tmp_server_names_lengths[roop_count] + 1; + if ((len= + sizeof(char) * share_alter->tmp_server_names_lengths[roop_count])) + { + tmp_server_names[roop_count]= tmp_server_names_char; + memcpy(tmp_server_names_char, share_alter->tmp_server_names[roop_count], + len); + tmp_server_names_char+= len + 1; + } - tmp_tgt_table_names[roop_count] = tmp_tgt_table_names_char; - memcpy(tmp_tgt_table_names_char, - share_alter->tmp_tgt_table_names[roop_count], - sizeof(char) * share_alter->tmp_tgt_table_names_lengths[roop_count]); - tmp_tgt_table_names_char += - share_alter->tmp_tgt_table_names_lengths[roop_count] + 1; + if ((len= sizeof(char) * + share_alter->tmp_tgt_table_names_lengths[roop_count])) + { + tmp_tgt_table_names[roop_count]= tmp_tgt_table_names_char; + memcpy(tmp_tgt_table_names_char, + share_alter->tmp_tgt_table_names[roop_count], len); + tmp_tgt_table_names_char+= len + 1; + } - tmp_tgt_dbs[roop_count] = tmp_tgt_dbs_char; - memcpy(tmp_tgt_dbs_char, share_alter->tmp_tgt_dbs[roop_count], - sizeof(char) * share_alter->tmp_tgt_dbs_lengths[roop_count]); - tmp_tgt_dbs_char += - share_alter->tmp_tgt_dbs_lengths[roop_count] + 1; + if ((len= sizeof(char) * share_alter->tmp_tgt_dbs_lengths[roop_count])) + { + tmp_tgt_dbs[roop_count]= tmp_tgt_dbs_char; + memcpy(tmp_tgt_dbs_char, share_alter->tmp_tgt_dbs[roop_count], len); + tmp_tgt_dbs_char+= len + 1; + } - tmp_tgt_hosts[roop_count] = tmp_tgt_hosts_char; - memcpy(tmp_tgt_hosts_char, share_alter->tmp_tgt_hosts[roop_count], - sizeof(char) * share_alter->tmp_tgt_hosts_lengths[roop_count]); - tmp_tgt_hosts_char += - share_alter->tmp_tgt_hosts_lengths[roop_count] + 1; + if ((len= sizeof(char) * share_alter->tmp_tgt_hosts_lengths[roop_count])) + { + tmp_tgt_hosts[roop_count]= tmp_tgt_hosts_char; + memcpy(tmp_tgt_hosts_char, share_alter->tmp_tgt_hosts[roop_count], len); + tmp_tgt_hosts_char+= len + 1; + } - tmp_tgt_usernames[roop_count] = tmp_tgt_usernames_char; - memcpy(tmp_tgt_usernames_char, share_alter->tmp_tgt_usernames[roop_count], - sizeof(char) * share_alter->tmp_tgt_usernames_lengths[roop_count]); - tmp_tgt_usernames_char += - share_alter->tmp_tgt_usernames_lengths[roop_count] + 1; + if ((len= sizeof(char) * + share_alter->tmp_tgt_usernames_lengths[roop_count])) + { + tmp_tgt_usernames[roop_count]= tmp_tgt_usernames_char; + memcpy(tmp_tgt_usernames_char, + share_alter->tmp_tgt_usernames[roop_count], len); + tmp_tgt_usernames_char+= len + 1; + } - tmp_tgt_passwords[roop_count] = tmp_tgt_passwords_char; - memcpy(tmp_tgt_passwords_char, share_alter->tmp_tgt_passwords[roop_count], - sizeof(char) * share_alter->tmp_tgt_passwords_lengths[roop_count]); - tmp_tgt_passwords_char += - share_alter->tmp_tgt_passwords_lengths[roop_count] + 1; + if ((len= sizeof(char) * + share_alter->tmp_tgt_passwords_lengths[roop_count])) + { + tmp_tgt_passwords[roop_count]= tmp_tgt_passwords_char; + memcpy(tmp_tgt_passwords_char, + share_alter->tmp_tgt_passwords[roop_count], len); + tmp_tgt_passwords_char+= len + 1; + } - tmp_tgt_sockets[roop_count] = tmp_tgt_sockets_char; - memcpy(tmp_tgt_sockets_char, share_alter->tmp_tgt_sockets[roop_count], - sizeof(char) * share_alter->tmp_tgt_sockets_lengths[roop_count]); - tmp_tgt_sockets_char += - share_alter->tmp_tgt_sockets_lengths[roop_count] + 1; + if ((len= sizeof(char) * share_alter->tmp_tgt_sockets_lengths[roop_count])) + { + tmp_tgt_sockets[roop_count]= tmp_tgt_sockets_char; + memcpy(tmp_tgt_sockets_char, share_alter->tmp_tgt_sockets[roop_count], + len); + tmp_tgt_sockets_char+= len + 1; + } - tmp_tgt_wrappers[roop_count] = tmp_tgt_wrappers_char; - memcpy(tmp_tgt_wrappers_char, share_alter->tmp_tgt_wrappers[roop_count], - sizeof(char) * share_alter->tmp_tgt_wrappers_lengths[roop_count]); - tmp_tgt_wrappers_char += - share_alter->tmp_tgt_wrappers_lengths[roop_count] + 1; + if ((len= + sizeof(char) * share_alter->tmp_tgt_wrappers_lengths[roop_count])) + { + tmp_tgt_wrappers[roop_count]= tmp_tgt_wrappers_char; + memcpy(tmp_tgt_wrappers_char, share_alter->tmp_tgt_wrappers[roop_count], + len); + tmp_tgt_wrappers_char+= len + 1; + } - tmp_tgt_ssl_cas[roop_count] = tmp_tgt_ssl_cas_char; - memcpy(tmp_tgt_ssl_cas_char, share_alter->tmp_tgt_ssl_cas[roop_count], - sizeof(char) * share_alter->tmp_tgt_ssl_cas_lengths[roop_count]); - tmp_tgt_ssl_cas_char += - share_alter->tmp_tgt_ssl_cas_lengths[roop_count] + 1; + if ((len= sizeof(char) * share_alter->tmp_tgt_ssl_cas_lengths[roop_count])) + { + tmp_tgt_ssl_cas[roop_count]= tmp_tgt_ssl_cas_char; + memcpy(tmp_tgt_ssl_cas_char, share_alter->tmp_tgt_ssl_cas[roop_count], + len); + tmp_tgt_ssl_cas_char+= len + 1; + } - tmp_tgt_ssl_capaths[roop_count] = tmp_tgt_ssl_capaths_char; - memcpy(tmp_tgt_ssl_capaths_char, - share_alter->tmp_tgt_ssl_capaths[roop_count], - sizeof(char) * share_alter->tmp_tgt_ssl_capaths_lengths[roop_count]); - tmp_tgt_ssl_capaths_char += - share_alter->tmp_tgt_ssl_capaths_lengths[roop_count] + 1; + if ((len= sizeof(char) * + share_alter->tmp_tgt_ssl_capaths_lengths[roop_count])) + { + tmp_tgt_ssl_capaths[roop_count]= tmp_tgt_ssl_capaths_char; + memcpy(tmp_tgt_ssl_capaths_char, + share_alter->tmp_tgt_ssl_capaths[roop_count], len); + tmp_tgt_ssl_capaths_char+= len + 1; + } - tmp_tgt_ssl_certs[roop_count] = tmp_tgt_ssl_certs_char; - memcpy(tmp_tgt_ssl_certs_char, share_alter->tmp_tgt_ssl_certs[roop_count], - sizeof(char) * share_alter->tmp_tgt_ssl_certs_lengths[roop_count]); - tmp_tgt_ssl_certs_char += - share_alter->tmp_tgt_ssl_certs_lengths[roop_count] + 1; + if ((len= sizeof(char) * + share_alter->tmp_tgt_ssl_certs_lengths[roop_count])) + { + tmp_tgt_ssl_certs[roop_count]= tmp_tgt_ssl_certs_char; + memcpy(tmp_tgt_ssl_certs_char, + share_alter->tmp_tgt_ssl_certs[roop_count], len); + tmp_tgt_ssl_certs_char+= len + 1; + } - tmp_tgt_ssl_ciphers[roop_count] = tmp_tgt_ssl_ciphers_char; - memcpy(tmp_tgt_ssl_ciphers_char, - share_alter->tmp_tgt_ssl_ciphers[roop_count], - sizeof(char) * share_alter->tmp_tgt_ssl_ciphers_lengths[roop_count]); - tmp_tgt_ssl_ciphers_char += - share_alter->tmp_tgt_ssl_ciphers_lengths[roop_count] + 1; + if ((len= sizeof(char) * + share_alter->tmp_tgt_ssl_ciphers_lengths[roop_count])) + { + tmp_tgt_ssl_ciphers[roop_count]= tmp_tgt_ssl_ciphers_char; + memcpy(tmp_tgt_ssl_ciphers_char, + share_alter->tmp_tgt_ssl_ciphers[roop_count], len); + tmp_tgt_ssl_ciphers_char+= len + 1; + } - tmp_tgt_ssl_keys[roop_count] = tmp_tgt_ssl_keys_char; - memcpy(tmp_tgt_ssl_keys_char, share_alter->tmp_tgt_ssl_keys[roop_count], - sizeof(char) * share_alter->tmp_tgt_ssl_keys_lengths[roop_count]); - tmp_tgt_ssl_keys_char += - share_alter->tmp_tgt_ssl_keys_lengths[roop_count] + 1; + if ((len= sizeof(char) * share_alter->tmp_tgt_ssl_keys_lengths[roop_count])) + { + tmp_tgt_ssl_keys[roop_count]= tmp_tgt_ssl_keys_char; + memcpy(tmp_tgt_ssl_keys_char, share_alter->tmp_tgt_ssl_keys[roop_count], + len); + tmp_tgt_ssl_keys_char+= len + 1; + } - tmp_tgt_default_files[roop_count] = tmp_tgt_default_files_char; - memcpy(tmp_tgt_default_files_char, - share_alter->tmp_tgt_default_files[roop_count], - sizeof(char) * share_alter->tmp_tgt_default_files_lengths[roop_count]); - tmp_tgt_default_files_char += - share_alter->tmp_tgt_default_files_lengths[roop_count] + 1; + if ((len= sizeof(char) * + share_alter->tmp_tgt_default_files_lengths[roop_count])) + { + tmp_tgt_default_files[roop_count]= tmp_tgt_default_files_char; + memcpy(tmp_tgt_default_files_char, + share_alter->tmp_tgt_default_files[roop_count], len); + tmp_tgt_default_files_char+= len + 1; + } - tmp_tgt_default_groups[roop_count] = tmp_tgt_default_groups_char; - memcpy(tmp_tgt_default_groups_char, - share_alter->tmp_tgt_default_groups[roop_count], - sizeof(char) * share_alter->tmp_tgt_default_groups_lengths[roop_count]); - tmp_tgt_default_groups_char += - share_alter->tmp_tgt_default_groups_lengths[roop_count] + 1; + if ((len= sizeof(char) * + share_alter->tmp_tgt_default_groups_lengths[roop_count])) + { + tmp_tgt_default_groups[roop_count]= tmp_tgt_default_groups_char; + memcpy(tmp_tgt_default_groups_char, + share_alter->tmp_tgt_default_groups[roop_count], len); + tmp_tgt_default_groups_char+= len + 1; + } - if (share_alter->tmp_static_link_ids[roop_count]) + if ((len= sizeof(char) * + share_alter->tmp_static_link_ids_lengths[roop_count])) { tmp_static_link_ids[roop_count] = tmp_static_link_ids_char; memcpy(tmp_static_link_ids_char, - share_alter->tmp_static_link_ids[roop_count], - sizeof(char) * share_alter->tmp_static_link_ids_lengths[roop_count]); - tmp_static_link_ids_char += - share_alter->tmp_static_link_ids_lengths[roop_count] + 1; + share_alter->tmp_static_link_ids[roop_count], len); + tmp_static_link_ids_char += len + 1; } } From ae96e21cf0a4696a3fb7ccb27de970ff2dc0dd6b Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Thu, 19 Jan 2023 09:25:40 +0100 Subject: [PATCH 4/6] Correct assert_grep.inc params in galera gcache tests --- mysql-test/suite/galera/t/galera_gcache_recover.test | 4 ++-- mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_gcache_recover.test b/mysql-test/suite/galera/t/galera_gcache_recover.test index e1bfe517d27..c6d19f308d6 100644 --- a/mysql-test/suite/galera/t/galera_gcache_recover.test +++ b/mysql-test/suite/galera/t/galera_gcache_recover.test @@ -59,7 +59,7 @@ CALL mtr.add_suppression("Skipped GCache ring buffer recovery"); --let $assert_select = async IST sender starting to serve --let $assert_count = 1 --let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err ---let $assert_only_after = starting as process +--let $assert_only_after = Starting MariaDB --source include/assert_grep.inc --connection node_2 @@ -71,7 +71,7 @@ CALL mtr.add_suppression("Skipped GCache ring buffer recovery"); --let $assert_select = Recovering GCache ring buffer: found gapless sequence --let $assert_count = 1 --let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err ---let $assert_only_after = starting as process +--let $assert_only_after = Starting MariaDB --source include/assert_grep.inc DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test b/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test index d92288b7881..50b01ec5206 100644 --- a/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test +++ b/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test @@ -211,7 +211,7 @@ CALL mtr.add_suppression("Skipped GCache ring buffer recovery"); --let $assert_select = async IST sender starting to serve --let $assert_count = 1 --let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err ---let $assert_only_after = starting as process +--let $assert_only_after = Starting MariaDB --source include/assert_grep.inc --connection node_2 @@ -224,5 +224,5 @@ CALL mtr.add_suppression("Skipped GCache ring buffer recovery"); --let $assert_select = Recovering GCache ring buffer: found gapless sequence --let $assert_count = 1 --let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err ---let $assert_only_after = starting as process +--let $assert_only_after = Starting MariaDB --source include/assert_grep.inc From eea9f2a1e7089f2b06faaabfedad0690b561f2ce Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 20 Jan 2023 09:52:00 +0400 Subject: [PATCH 5/6] MDEV-27653 long uniques don't work with unicode collations There are no source code changes in this commit! This is an empty follow-up commit for 284ac6f2b73650f138064c97a96c8e1d8846550b to comment what was done, as the patch itself did not have change comments. Problems solved in this patch: 1. The function calc_hash_for_unique() erroneously takes into account the string length, so equal strings (in terms of the collation) with different lengths got different hash value. For example: - LATIN LETTER A - 1 byte - LATIN LETTER A WITH ACUTE - 2 bytes are equal in utf8_general_ci, but as their lengths are different, calc_hash_for_unique() returned different hash values. 2. calc_hash_for_unique() also erroneously used val_str() result to calculate hashes. This may not be correct for some data types, e.g. TIMESTAMP, as its string value depends on the session environment (e.g. @@time_zone). Change summary: Instead of doing Item::val_str(), we should always call Field::hash() of the underlying Field. It properly handles both cases (equal strings with different lengths, as well as tricky data types like TIMESTAMP). Detailed change description: Non-functional changes (make the code cleaner): - Adding a helper class Hasher, to pass hash parts nr1 and nr2 through function arguments easier. - Splitting virtual Field::hash() into non-virtual wrapper Field::hash() and virtual Field::hash_not_null(). This helps to get rid of duplicate code handling SQL NULL, as it was equal in all Field_xxx implementations. - Adding a new method THD::my_ok_with_recreate_info(). Actual fix changes (make new tables work properly): - Adding a virtual method Item::hash_not_null() This helps to handle hashes on full fields (Item_field) and hashes on prefix fields (Item_func_left(Item_field)) in a polymorphic way. Implementing overrides for Item_field and Item_func_left. - Rewriting Item_func_hash::val_int() to use Item::hash_not_null(), instead of the combination of val_str() and alc_hash_for_unique(). Backward compatibility changes (make old tables work in the new server): - Adding a new class Item_func_hash_mariadb_100403. Moving the old version of Item_func_hash::val_int() into Item_func_hash_mariadb_100403::val_int(). The old class Item_func_hash_mariadb_100403 is still needed, to open old tables before upgrade is done. - Adding TABLE_SHARE::old_long_hash_function() and handler::check_long_hash_compatibility() to test if a table is using an old hash function. - Adding a helper method TABLE_SHARE::make_long_hash_func() to instantiate either Item_func_hash_mariadb_100403 (for old not upgraded tables) or Item_func_hash (for new tables). Upgrade changes (make old tables upgrade in the new server properly): Upgrading an old table to a new hash can be done using either of these two statements: ALTER IGNORE TABLE t1 FORCE; REPAIR TABLE t1; !!! These statements find and filter out erreneous duplicates!!! The table after these statements will have less records if there were erroneous duplicates (such and A and A WITH ACUTE). The information about filtered out records is reported in both statements. - Adding a new class Recreate_info to return out information about copied and duplucate rows from these functions: - mysql_alter_table() - mysql_recreate_table() - admin_recreate_table() This helps to print a warning during REPAIR: MariaDB [test]> repair table mdev27653_100422_text; +----------------------------+--------+----------+------------------------------------+ | Table | Op | Msg_type | Msg_text | +----------------------------+--------+----------+------------------------------------+ | test.mdev27653_100422_text | repair | Warning | Number of rows changed from 2 to 1 | | test.mdev27653_100422_text | repair | status | OK | +----------------------------+--------+----------+------------------------------------+ 2 rows in set (0.018 sec) From b2b9d91668ca7af9bbed95c5b3f502c39c9020af Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Thu, 12 Jan 2023 13:18:33 +0700 Subject: [PATCH 6/6] MDEV-29294 Assertion `functype() == ((Item_cond *) new_item)->functype()' failed in Item_cond::remove_eq_conds on SELECT Item_singlerow_subselect may be converted to Item_cond during optimization. So there is a possibility of constructing nested Item_cond_and or Item_cond_or which is not allowed (such conditions must be flattened). This commit checks if such kind of optimization has been applied and flattens the condition if needed --- mysql-test/main/select.result | 56 +++++++++++++++++ mysql-test/main/select.test | 40 ++++++++++++ mysql-test/main/select_jcl6.result | 56 +++++++++++++++++ mysql-test/main/select_pkeycache.result | 56 +++++++++++++++++ sql/item_cmpfunc.cc | 83 +++++++++++++++++-------- sql/item_cmpfunc.h | 3 + 6 files changed, 268 insertions(+), 26 deletions(-) diff --git a/mysql-test/main/select.result b/mysql-test/main/select.result index 1562144b164..fc3a29094ae 100644 --- a/mysql-test/main/select.result +++ b/mysql-test/main/select.result @@ -5639,4 +5639,60 @@ EXECUTE stmt; COUNT(DISTINCT a) 3 DROP TABLE t1; +# +# MDEV-29294: Assertion `functype() == ((Item_cond *) new_item)->functype()' +# failed in Item_cond::remove_eq_conds on SELECT +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2),(3); +# Test for nested OR conditions: +SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +a +1 +EXPLAIN EXTENDED +SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where; Using temporary +3 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1276 Field or reference 'test.t1.a' of SELECT #2 was resolved in SELECT #1 +Note 1276 Field or reference 'test.t1.a' of SELECT #3 was resolved in SELECT #1 +Note 1276 Field or reference 'test.t1.a' of SELECT #3 was resolved in SELECT #1 +Note 1249 Select 2 was reduced during optimization +Note 1003 /* select#1 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = 1 and (1 or <`test`.`t1`.`a`>((/* select#3 */ select 3 from DUAL where `test`.`t1`.`a` = `test`.`t1`.`a`)) = 3) +PREPARE stmt FROM 'SELECT * FROM t1 WHERE a = 1 AND + (3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3))'; +EXECUTE stmt; +a +1 +EXECUTE stmt; +a +1 +CREATE VIEW v1 AS SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +SELECT * FROM v1; +a +1 +# Test for nested AND conditions: +SELECT * FROM t1 WHERE a = 1 OR +(3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3)); +a +1 +PREPARE stmt FROM 'SELECT * FROM t1 WHERE a = 1 OR + (3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3))'; +EXECUTE stmt; +a +1 +EXECUTE stmt; +a +1 +CREATE VIEW v2 AS SELECT * FROM t1 WHERE a = 1 OR +(3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3)); +SELECT * FROM v2; +a +1 +DROP TABLE t1; +DROP VIEW v1, v2; End of 10.0 tests diff --git a/mysql-test/main/select.test b/mysql-test/main/select.test index 31487a6844e..ed8783d40f4 100644 --- a/mysql-test/main/select.test +++ b/mysql-test/main/select.test @@ -4742,4 +4742,44 @@ EXECUTE stmt; --enable_warnings DROP TABLE t1; +--echo # +--echo # MDEV-29294: Assertion `functype() == ((Item_cond *) new_item)->functype()' +--echo # failed in Item_cond::remove_eq_conds on SELECT +--echo # +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2),(3); + +--echo # Test for nested OR conditions: +SELECT * FROM t1 WHERE a = 1 AND + (3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); + +EXPLAIN EXTENDED +SELECT * FROM t1 WHERE a = 1 AND + (3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); + +PREPARE stmt FROM 'SELECT * FROM t1 WHERE a = 1 AND + (3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3))'; +EXECUTE stmt; +EXECUTE stmt; + +CREATE VIEW v1 AS SELECT * FROM t1 WHERE a = 1 AND + (3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +SELECT * FROM v1; + +--echo # Test for nested AND conditions: +SELECT * FROM t1 WHERE a = 1 OR + (3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3)); + +PREPARE stmt FROM 'SELECT * FROM t1 WHERE a = 1 OR + (3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3))'; +EXECUTE stmt; +EXECUTE stmt; + +CREATE VIEW v2 AS SELECT * FROM t1 WHERE a = 1 OR + (3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3)); +SELECT * FROM v2; + +DROP TABLE t1; +DROP VIEW v1, v2; + --echo End of 10.0 tests diff --git a/mysql-test/main/select_jcl6.result b/mysql-test/main/select_jcl6.result index e144477b66e..e7c2e3e4807 100644 --- a/mysql-test/main/select_jcl6.result +++ b/mysql-test/main/select_jcl6.result @@ -5650,6 +5650,62 @@ EXECUTE stmt; COUNT(DISTINCT a) 3 DROP TABLE t1; +# +# MDEV-29294: Assertion `functype() == ((Item_cond *) new_item)->functype()' +# failed in Item_cond::remove_eq_conds on SELECT +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2),(3); +# Test for nested OR conditions: +SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +a +1 +EXPLAIN EXTENDED +SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where; Using temporary +3 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1276 Field or reference 'test.t1.a' of SELECT #2 was resolved in SELECT #1 +Note 1276 Field or reference 'test.t1.a' of SELECT #3 was resolved in SELECT #1 +Note 1276 Field or reference 'test.t1.a' of SELECT #3 was resolved in SELECT #1 +Note 1249 Select 2 was reduced during optimization +Note 1003 /* select#1 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = 1 and (1 or <`test`.`t1`.`a`>((/* select#3 */ select 3 from DUAL where `test`.`t1`.`a` = `test`.`t1`.`a`)) = 3) +PREPARE stmt FROM 'SELECT * FROM t1 WHERE a = 1 AND + (3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3))'; +EXECUTE stmt; +a +1 +EXECUTE stmt; +a +1 +CREATE VIEW v1 AS SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +SELECT * FROM v1; +a +1 +# Test for nested AND conditions: +SELECT * FROM t1 WHERE a = 1 OR +(3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3)); +a +1 +PREPARE stmt FROM 'SELECT * FROM t1 WHERE a = 1 OR + (3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3))'; +EXECUTE stmt; +a +1 +EXECUTE stmt; +a +1 +CREATE VIEW v2 AS SELECT * FROM t1 WHERE a = 1 OR +(3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3)); +SELECT * FROM v2; +a +1 +DROP TABLE t1; +DROP VIEW v1, v2; End of 10.0 tests set join_cache_level=default; set @@optimizer_switch=@save_optimizer_switch_jcl6; diff --git a/mysql-test/main/select_pkeycache.result b/mysql-test/main/select_pkeycache.result index 1562144b164..fc3a29094ae 100644 --- a/mysql-test/main/select_pkeycache.result +++ b/mysql-test/main/select_pkeycache.result @@ -5639,4 +5639,60 @@ EXECUTE stmt; COUNT(DISTINCT a) 3 DROP TABLE t1; +# +# MDEV-29294: Assertion `functype() == ((Item_cond *) new_item)->functype()' +# failed in Item_cond::remove_eq_conds on SELECT +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2),(3); +# Test for nested OR conditions: +SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +a +1 +EXPLAIN EXTENDED +SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where; Using temporary +3 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1276 Field or reference 'test.t1.a' of SELECT #2 was resolved in SELECT #1 +Note 1276 Field or reference 'test.t1.a' of SELECT #3 was resolved in SELECT #1 +Note 1276 Field or reference 'test.t1.a' of SELECT #3 was resolved in SELECT #1 +Note 1249 Select 2 was reduced during optimization +Note 1003 /* select#1 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = 1 and (1 or <`test`.`t1`.`a`>((/* select#3 */ select 3 from DUAL where `test`.`t1`.`a` = `test`.`t1`.`a`)) = 3) +PREPARE stmt FROM 'SELECT * FROM t1 WHERE a = 1 AND + (3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3))'; +EXECUTE stmt; +a +1 +EXECUTE stmt; +a +1 +CREATE VIEW v1 AS SELECT * FROM t1 WHERE a = 1 AND +(3 = 0 OR (SELECT a = 1 OR (SELECT 3 WHERE a = a) = 3)); +SELECT * FROM v1; +a +1 +# Test for nested AND conditions: +SELECT * FROM t1 WHERE a = 1 OR +(3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3)); +a +1 +PREPARE stmt FROM 'SELECT * FROM t1 WHERE a = 1 OR + (3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3))'; +EXECUTE stmt; +a +1 +EXECUTE stmt; +a +1 +CREATE VIEW v2 AS SELECT * FROM t1 WHERE a = 1 OR +(3 = 3 AND (SELECT a = 1 AND (SELECT 3 WHERE a = a) = 3)); +SELECT * FROM v2; +a +1 +DROP TABLE t1; +DROP VIEW v1, v2; End of 10.0 tests diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 7fb2ff32842..6fbbb79c263 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4816,38 +4816,18 @@ Item_cond::fix_fields(THD *thd, Item **ref) if (check_stack_overrun(thd, STACK_MIN_SIZE, buff)) return TRUE; // Fatal error flag is set! - /* - The following optimization reduces the depth of an AND-OR tree. - E.g. a WHERE clause like - F1 AND (F2 AND (F2 AND F4)) - is parsed into a tree with the same nested structure as defined - by braces. This optimization will transform such tree into - AND (F1, F2, F3, F4). - Trees of OR items are flattened as well: - ((F1 OR F2) OR (F3 OR F4)) => OR (F1, F2, F3, F4) - Items for removed AND/OR levels will dangle until the death of the - entire statement. - The optimization is currently prepared statements and stored procedures - friendly as it doesn't allocate any memory and its effects are durable - (i.e. do not depend on PS/SP arguments). - */ - while ((item=li++)) + + while (li++) { - while (item->type() == Item::COND_ITEM && - ((Item_cond*) item)->functype() == functype() && - !((Item_cond*) item)->list.is_empty()) - { // Identical function - li.replace(((Item_cond*) item)->list); - ((Item_cond*) item)->list.empty(); - item= *li.ref(); // new current item - } + merge_sub_condition(li); + item= *li.ref(); if (abort_on_null) item->top_level_item(); /* replace degraded condition: was: - become: = 1 + become: != 0 */ Item::Type type= item->type(); if (type == Item::FIELD_ITEM || type == Item::REF_ITEM) @@ -4863,7 +4843,9 @@ Item_cond::fix_fields(THD *thd, Item **ref) if (item->fix_fields_if_needed_for_bool(thd, li.ref())) return TRUE; /* purecov: inspected */ - item= *li.ref(); // item can be substituted in fix_fields + merge_sub_condition(li); + item= *li.ref(); // may be substituted in fix_fields/merge_item_if_possible + used_tables_cache|= item->used_tables(); if (item->const_item() && !item->with_param && !item->is_expensive() && !cond_has_datetime_is_null(item)) @@ -4915,6 +4897,55 @@ Item_cond::fix_fields(THD *thd, Item **ref) return FALSE; } +/** + @brief + Merge a lower-level condition pointed by the iterator into this Item_cond + if possible + + @param li list iterator pointing to condition that must be + examined and merged if possible. + + @details + If an item pointed by the iterator is an instance of Item_cond with the + same functype() as this Item_cond (i.e. both are Item_cond_and or both are + Item_cond_or) then the arguments of that lower-level item can be merged + into the list of arguments of this upper-level Item_cond. + + This optimization reduces the depth of an AND-OR tree. + E.g. a WHERE clause like + F1 AND (F2 AND (F2 AND F4)) + is parsed into a tree with the same nested structure as defined + by braces. This optimization will transform such tree into + AND (F1, F2, F3, F4). + Trees of OR items are flattened as well: + ((F1 OR F2) OR (F3 OR F4)) => OR (F1, F2, F3, F4) + Items for removed AND/OR levels will dangle until the death of the + entire statement. + + The optimization is currently prepared statements and stored procedures + friendly as it doesn't allocate any memory and its effects are durable + (i.e. do not depend on PS/SP arguments). +*/ +void Item_cond::merge_sub_condition(List_iterator& li) +{ + Item *item= *li.ref(); + + /* + The check for list.is_empty() is to catch empty Item_cond_and() items. + We may encounter Item_cond_and with an empty list, because optimizer code + strips multiple equalities, combines items, then adds multiple equalities + back + */ + while (item->type() == Item::COND_ITEM && + ((Item_cond*) item)->functype() == functype() && + !((Item_cond*) item)->list.is_empty()) + { + li.replace(((Item_cond*) item)->list); + ((Item_cond*) item)->list.empty(); + item= *li.ref(); + } +} + bool Item_cond::eval_not_null_tables(void *opt_arg) diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 01834fe06d7..0641213c7c9 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -3046,6 +3046,9 @@ public: Item *build_clone(THD *thd); bool excl_dep_on_table(table_map tab_map); bool excl_dep_on_grouping_fields(st_select_lex *sel); + +private: + void merge_sub_condition(List_iterator& li); }; template class LI, class T> class Item_equal_iterator;