From 57ae4992ed8f5e1f381c462774447c3a41d23739 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 6 Feb 2018 11:29:36 -0500 Subject: [PATCH 01/19] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 24d355ca91e..75daccdd9d4 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=0 -MYSQL_VERSION_PATCH=34 +MYSQL_VERSION_PATCH=35 From 7bd95307f2aa901c141d4d6c4c568a2ed85ba01b Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Tue, 30 Jan 2018 05:37:22 -0800 Subject: [PATCH 02/19] Bump wsrep patch version to 25.23 --- cmake/wsrep.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index cc6627d1eea..2ba32b469e0 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "21") +SET(WSREP_PATCH_VERSION "23") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. From 26e4a48bda8a09ac4ddef64e12841fbee29f4d7d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 10:50:38 +1100 Subject: [PATCH 03/19] MDEV-8743: ib_logfile0 Use O_CLOEXEC so galera SST scripts don't get fd --- storage/innobase/os/os0file.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index e946bcf5f30..df096dcc6fd 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3578,7 +3578,7 @@ os_aio_native_aio_supported(void) strcpy(name + dirnamelen, "ib_logfile0"); - fd = ::open(name, O_RDONLY); + fd = ::open(name, O_RDONLY | O_CLOEXEC); if (fd == -1) { From 8b54c314863e7e9861470844e09c0453800748ae Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 14 Mar 2018 13:31:28 +1100 Subject: [PATCH 04/19] MDEV-8743: where O_CLOEXEC is available, use for innodb buf_dump As this is the only moderately critical fopened for writing file, create an alternate path to use open and fdopen for non-glibc platforms that support O_CLOEXEC (BSDs). Tested on Linux (by modifing the GLIBC defination) to take this alternate path: $ cd /proc/23874 $ more fdinfo/71 pos: 0 flags: 02100001 mnt_id: 24 $ ls -la fd/71 l-wx------. 1 dan dan 64 Mar 14 13:30 fd/71 -> /dev/shm/var_auto_i7rl/mysqld.1/data/ib_buffer_pool.incomplete --- include/my_global.h | 5 +++++ storage/innobase/buf/buf0dump.cc | 15 ++++++++++++++- storage/xtradb/buf/buf0dump.cc | 15 ++++++++++++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/include/my_global.h b/include/my_global.h index 5ea761d587e..ad1527d7b37 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -601,6 +601,11 @@ typedef SOCKET_SIZE_TYPE size_socket; #ifndef O_CLOEXEC #define O_CLOEXEC 0 #endif +#ifdef __GLIBC__ +#define STR_O_CLOEXEC "e" +#else +#define STR_O_CLOEXEC "" +#endif #ifndef SOCK_CLOEXEC #define SOCK_CLOEXEC 0 #endif diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 5d5be0505f6..306dcc2ec18 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -220,7 +220,20 @@ buf_dump( buf_dump_status(STATUS_NOTICE, "Dumping buffer pool(s) to %s", full_filename); - f = fopen(tmp_filename, "w"); +#if defined(__GLIBC__) || defined(__WIN__) || O_CLOEXEC == 0 + f = fopen(tmp_filename, "w" STR_O_CLOEXEC); +#else + { + int fd; + fd = open(tmp_filename, O_CREAT | O_TRUNC | O_CLOEXEC | O_WRONLY, 0640); + if (fd >= 0) { + f = fdopen(fd, "w"); + } + else { + f = NULL; + } + } +#endif if (f == NULL) { buf_dump_status(STATUS_ERR, "Cannot open '%s' for writing: %s", diff --git a/storage/xtradb/buf/buf0dump.cc b/storage/xtradb/buf/buf0dump.cc index 09ac460f865..0bc222d2cc2 100644 --- a/storage/xtradb/buf/buf0dump.cc +++ b/storage/xtradb/buf/buf0dump.cc @@ -220,7 +220,20 @@ buf_dump( buf_dump_status(STATUS_NOTICE, "Dumping buffer pool(s) to %s", full_filename); - f = fopen(tmp_filename, "w"); +#if defined(__GLIBC__) || defined(__WIN__) || O_CLOEXEC == 0 + f = fopen(tmp_filename, "w" STR_O_CLOEXEC); +#else + { + int fd; + fd = open(tmp_filename, O_CREAT | O_TRUNC | O_CLOEXEC | O_WRONLY, 0640); + if (fd >= 0) { + f = fdopen(fd, "w"); + } + else { + f = NULL; + } + } +#endif if (f == NULL) { buf_dump_status(STATUS_ERR, "Cannot open '%s' for writing: %s", From a0c722d853071e605ea025168da1b01bfe21092c Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Fri, 16 Mar 2018 18:35:41 +0530 Subject: [PATCH 05/19] MDEV-15321:different results when using value of optimizer_use_condition_selectivity=4 and =1 To disallow equality propagation for DATETIME with non-zero YYYYMMDD part we were setting null_value to true. This caused issues when we were calculating selectivity for a condition as this returned IMPOSSIBLE WHERE. The issue is resolved by not setting null_value to true for DATETIME with non-zero YYYYMMDD. --- mysql-test/r/type_time.result | 31 +++++++++++++++++++++++++++++++ mysql-test/t/type_time.test | 23 +++++++++++++++++++++++ sql/item.cc | 14 ++++++++------ 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/type_time.result b/mysql-test/r/type_time.result index b5689d31aef..fc287e86fbd 100644 --- a/mysql-test/r/type_time.result +++ b/mysql-test/r/type_time.result @@ -1215,5 +1215,36 @@ MAX(a) MAX(COALESCE(a)) 10:20:30 10:20:30 DROP TABLE t1; # +# MDEV-15321: different results when using value of optimizer_use_condition_selectivity=4 and =1 +# +SET @save_old_mode=@@old_mode; +SET @@old_mode=zero_date_time_cast; +CREATE TABLE t1 (a TIME); +INSERT INTO t1 VALUES ('0000-00-00 10:20:30'),('0000-00-00 10:20:31'); +INSERT INTO t1 VALUES ('0000-00-01 10:20:30'),('0000-00-01 10:20:31'); +INSERT INTO t1 VALUES ('31 10:20:30'),('32 10:20:30'),('33 10:20:30'),('34 10:20:30'); +SET @save_optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; +SET @@optimizer_use_condition_selectivity=1; +SELECT * FROM t1 WHERE a='0000-00-01 10:20:30' AND LENGTH(a)=8; +a +34:20:30 +EXPLAIN EXTENDED SELECT * FROM t1 WHERE a='0000-00-01 10:20:30' AND LENGTH(a)=8; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where ((`test`.`t1`.`a` = '0000-00-01 10:20:30') and (length(`test`.`t1`.`a`) = 8)) +SET @@optimizer_use_condition_selectivity=4; +SELECT * FROM t1 WHERE a='0000-00-01 10:20:30' AND LENGTH(a)=8; +a +34:20:30 +EXPLAIN EXTENDED SELECT * FROM t1 WHERE a='0000-00-01 10:20:30' AND LENGTH(a)=8; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where ((`test`.`t1`.`a` = '0000-00-01 10:20:30') and (length(`test`.`t1`.`a`) = 8)) +drop table t1; +SET @@optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +set @@old_mode= @save_old_mode; +# # End of 10.1 tests # diff --git a/mysql-test/t/type_time.test b/mysql-test/t/type_time.test index 27679b9ec5a..10b75f27546 100644 --- a/mysql-test/t/type_time.test +++ b/mysql-test/t/type_time.test @@ -723,6 +723,29 @@ INSERT INTO t1 VALUES ('10:10:10'),('10:20:30'); SELECT MAX(a), MAX(COALESCE(a)) FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-15321: different results when using value of optimizer_use_condition_selectivity=4 and =1 +--echo # + +SET @save_old_mode=@@old_mode; +SET @@old_mode=zero_date_time_cast; +CREATE TABLE t1 (a TIME); +INSERT INTO t1 VALUES ('0000-00-00 10:20:30'),('0000-00-00 10:20:31'); +INSERT INTO t1 VALUES ('0000-00-01 10:20:30'),('0000-00-01 10:20:31'); +INSERT INTO t1 VALUES ('31 10:20:30'),('32 10:20:30'),('33 10:20:30'),('34 10:20:30'); + +SET @save_optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; +SET @@optimizer_use_condition_selectivity=1; +SELECT * FROM t1 WHERE a='0000-00-01 10:20:30' AND LENGTH(a)=8; +EXPLAIN EXTENDED SELECT * FROM t1 WHERE a='0000-00-01 10:20:30' AND LENGTH(a)=8; + +SET @@optimizer_use_condition_selectivity=4; +SELECT * FROM t1 WHERE a='0000-00-01 10:20:30' AND LENGTH(a)=8; +EXPLAIN EXTENDED SELECT * FROM t1 WHERE a='0000-00-01 10:20:30' AND LENGTH(a)=8; +drop table t1; +SET @@optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +set @@old_mode= @save_old_mode; + --echo # --echo # End of 10.1 tests --echo # diff --git a/sql/item.cc b/sql/item.cc index 2cc45dc6010..1d0ed6a6ea5 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -166,19 +166,21 @@ bool Item::get_time_with_conversion(THD *thd, MYSQL_TIME *ltime, - truncate the YYYYMMDD part - add (MM*33+DD)*24 to hours - add (MM*31+DD)*24 to hours - Let's return NULL here, to disallow equal field propagation. + Let's return TRUE here, to disallow equal field propagation. Note, If we start to use this method in more pieces of the code other - than eqial field propagation, we should probably return - NULL only if some flag in fuzzydate is set. + than equal field propagation, we should probably return + TRUE only if some flag in fuzzydate is set. */ - return (null_value= true); + return true; } if (datetime_to_time_with_warn(thd, ltime, <ime2, TIME_SECOND_PART_DIGITS)) { /* - Time difference between CURRENT_DATE and ltime - did not fit into the supported TIME range + If the time difference between CURRENT_DATE and ltime + did not fit into the supported TIME range, then we set the + difference to the maximum possible value in the supported TIME range */ + DBUG_ASSERT(0); return (null_value= true); } *ltime= ltime2; From e5e83249c15927576d0d6066bfce32d5a5e103c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 15 Feb 2018 10:34:01 +0200 Subject: [PATCH 06/19] MDEV-14875: galera_new_cluster crashes mysqld when existing server contains databases Fortify wsrep_hton so that wsrep calls are not done to NULL-pointers. --- sql/wsrep_hton.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc index 75c1526cb15..944c8d83b76 100644 --- a/sql/wsrep_hton.cc +++ b/sql/wsrep_hton.cc @@ -120,7 +120,7 @@ void wsrep_post_commit(THD* thd, bool all) case LOCAL_COMMIT: { DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); - if (wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) + if (wsrep && wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) { DBUG_PRINT("wsrep", ("set committed fail")); WSREP_WARN("set committed fail: %llu %d", @@ -252,7 +252,7 @@ static int wsrep_rollback(handlerton *hton, THD *thd, bool all) if ((all || !thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) && (thd->variables.wsrep_on && thd->wsrep_conflict_state != MUST_REPLAY)) { - if (wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) + if (wsrep && wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) { DBUG_PRINT("wsrep", ("setting rollback fail")); WSREP_ERROR("settting rollback fail: thd: %llu, schema: %s, SQL: %s", @@ -294,7 +294,7 @@ int wsrep_commit(handlerton *hton, THD *thd, bool all) possible changes to clean state. */ if (WSREP_PROVIDER_EXISTS) { - if (wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) + if (wsrep && wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) { DBUG_PRINT("wsrep", ("setting rollback fail")); WSREP_ERROR("settting rollback fail: thd: %llu, schema: %s, SQL: %s", @@ -471,7 +471,7 @@ wsrep_run_wsrep_commit(THD *thd, bool all) } else if (!rcode) { - if (WSREP_OK == rcode) + if (WSREP_OK == rcode && wsrep) rcode = wsrep->pre_commit(wsrep, (wsrep_conn_id_t)thd->thread_id, &thd->wsrep_ws_handle, From f46155a31b97b3458fb6748e945c223c901f642d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 14 Feb 2018 13:52:37 +0200 Subject: [PATCH 07/19] MDEV-13549: Galera test failures Fix test failure on galera_concurrent_ctas --- mysql-test/suite/galera/disabled.def | 1 - mysql-test/suite/galera/t/galera_concurrent_ctas.test | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 6d4574c0403..19105e88881 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -30,7 +30,6 @@ galera_gcache_recover_manytrx : MDEV-13549 Galera test failures galera_ist_mysqldump : MDEV-13549 Galera test failures mysql-wsrep#31 : MDEV-13549 Galera test failures galera_migrate : MariaDB 10.0 does not support START SLAVE USER -galera_concurrent_ctas : MDEV-13549 Galera test failures galera_wsrep_desync_wsrep_on : MDEV-13549 Galera test failures galera_ssl_upgrade : MDEV-13549 Galera test failures mysql-wsrep#33 : MDEV-13549 Galera test failures diff --git a/mysql-test/suite/galera/t/galera_concurrent_ctas.test b/mysql-test/suite/galera/t/galera_concurrent_ctas.test index f0dcf8e4900..6c4e8be68a7 100644 --- a/mysql-test/suite/galera/t/galera_concurrent_ctas.test +++ b/mysql-test/suite/galera/t/galera_concurrent_ctas.test @@ -43,9 +43,9 @@ let $run=10; while($run) { --error 0,1 - exec $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_1 test - < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql & - $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_2 test + exec $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_1 test \ + < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql & \ + $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_2 test \ < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql; dec $run; } From 31e2ab513d1d0d6caad96f613cbce3ad25c19497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 14 Feb 2018 09:48:29 +0200 Subject: [PATCH 08/19] MDEV-13549: Galera test failures Fix test failure on galera_flush_local. --- mysql-test/suite/galera/disabled.def | 1 - .../suite/galera/include/have_wsrep_replicate_myisam.inc | 4 ++++ mysql-test/suite/galera/r/have_wsrep_replicate_myisam.require | 2 ++ mysql-test/suite/galera/t/galera_flush_local.opt | 4 +++- mysql-test/suite/galera/t/galera_flush_local.test | 1 + 5 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/galera/include/have_wsrep_replicate_myisam.inc create mode 100644 mysql-test/suite/galera/r/have_wsrep_replicate_myisam.require diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 19105e88881..da268abe0d2 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -10,7 +10,6 @@ # ############################################################################## galera_gra_log : MDEV-13549 Galera test failures -galera_flush_local : MDEV-13549 Galera test failures galera_flush : MDEV-13549 Galera test failures galera_account_management : MariaDB 10.0 does not support ALTER USER galera_binlog_row_image : MariaDB 10.0 does not support binlog_row_image diff --git a/mysql-test/suite/galera/include/have_wsrep_replicate_myisam.inc b/mysql-test/suite/galera/include/have_wsrep_replicate_myisam.inc new file mode 100644 index 00000000000..726fc6e2b18 --- /dev/null +++ b/mysql-test/suite/galera/include/have_wsrep_replicate_myisam.inc @@ -0,0 +1,4 @@ +--require suite/galera/r/have_wsrep_replicate_myisam.require +disable_query_log; +SHOW VARIABLES LIKE 'wsrep_replicate_myisam'; +enable_query_log; diff --git a/mysql-test/suite/galera/r/have_wsrep_replicate_myisam.require b/mysql-test/suite/galera/r/have_wsrep_replicate_myisam.require new file mode 100644 index 00000000000..c55610fd049 --- /dev/null +++ b/mysql-test/suite/galera/r/have_wsrep_replicate_myisam.require @@ -0,0 +1,2 @@ +Variable_name Value +wsrep_replicate_myisam ON diff --git a/mysql-test/suite/galera/t/galera_flush_local.opt b/mysql-test/suite/galera/t/galera_flush_local.opt index 5a1fb6748d9..a084db15c5d 100644 --- a/mysql-test/suite/galera/t/galera_flush_local.opt +++ b/mysql-test/suite/galera/t/galera_flush_local.opt @@ -1 +1,3 @@ ---query_cache_type=1 --query_cache_size=1000000 +--query_cache_type=1 +--query_cache_size=1000000 +--wsrep_replicate_myisam=ON diff --git a/mysql-test/suite/galera/t/galera_flush_local.test b/mysql-test/suite/galera/t/galera_flush_local.test index 768f4ea4f1b..24acd9ec4ff 100644 --- a/mysql-test/suite/galera/t/galera_flush_local.test +++ b/mysql-test/suite/galera/t/galera_flush_local.test @@ -5,6 +5,7 @@ --source include/galera_cluster.inc --source include/have_innodb.inc --source include/have_query_cache.inc +--source include/have_wsrep_replicate_myisam.inc --disable_warnings DROP TABLE IF EXISTS t1, t2, x1, x2; From f538a64817ce583fcce558303ae9d9b6aeecf838 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 19 Mar 2018 13:07:41 +0400 Subject: [PATCH 09/19] MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple cmp_item_sort_string::store_value() did not cache the string returned from item->val_str(), whose result can point to various private members such as Item_char_typecast::tmp_value. - cmp_item_sort_string::store_value() remembered the pointer returned from item->val_str() poiting to tmp_value into cmp_item_string::value_res. - Later, cmp_item_real::store_value() was called, which called Item_str_func::val_real(), which called Item_char_typecast::val_str(&tmp) using a local stack variable "String tmp". Item_char_typecast::tmp_value was overwritten and become a link to "tmp": tmp_value.Ptr freed its own buffer and set to point to the buffer owned by "tmp". - On return from Item_str_func::val_real(), "String tmp" was destructed, but "tmp_value" still pointed to the buffer owned by "tmp", So tmp_value.Ptr became invalid. - Then cmp_item_sort_string() passed cmp_item_string::value_res to sortcmp(). At this point, value_res still pointed to an invalid value of Item_char_typecast::tmp_value. Fix: changing cmp_item_sort_string::store_value() to force copying to cmp_item_string::value if item->val_str(&value) returned a different pointer (instead of &value). --- mysql-test/r/ctype_latin1.result | 13 +++++++++++++ mysql-test/r/ctype_utf8.result | 13 +++++++++++++ mysql-test/t/ctype_latin1.test | 9 +++++++++ mysql-test/t/ctype_utf8.test | 10 ++++++++++ sql/item_cmpfunc.h | 7 +++++++ 5 files changed, 52 insertions(+) diff --git a/mysql-test/r/ctype_latin1.result b/mysql-test/r/ctype_latin1.result index 85035982cf9..66c5a37750d 100644 --- a/mysql-test/r/ctype_latin1.result +++ b/mysql-test/r/ctype_latin1.result @@ -8208,5 +8208,18 @@ Warnings: Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c`,`test`.`t1`.`d` AS `d` from `test`.`t1` where ((coalesce(`test`.`t1`.`c`,0) = '3 ') and (coalesce(`test`.`t1`.`d`,0) = '3 ')) DROP TABLE t1; # +# MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple +# +SET NAMES latin1; +SELECT CONVERT(1, CHAR) IN ('100', 10, '101'); +CONVERT(1, CHAR) IN ('100', 10, '101') +0 +SELECT CONVERT(1, CHAR) IN ('100', 10, '1'); +CONVERT(1, CHAR) IN ('100', 10, '1') +1 +SELECT CONVERT(1, CHAR) IN ('100', '10', '1'); +CONVERT(1, CHAR) IN ('100', '10', '1') +1 +# # End of 10.1 tests # diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index 5a77ea3aea5..3da74d12877 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -10535,5 +10535,18 @@ CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1) Warnings: Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' # +# MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple +# +SET NAMES utf8; +SELECT CONVERT(1, CHAR) IN ('100', 10, '101'); +CONVERT(1, CHAR) IN ('100', 10, '101') +0 +SELECT CONVERT(1, CHAR) IN ('100', 10, '1'); +CONVERT(1, CHAR) IN ('100', 10, '1') +1 +SELECT CONVERT(1, CHAR) IN ('100', '10', '1'); +CONVERT(1, CHAR) IN ('100', '10', '1') +1 +# # End of 10.1 tests # diff --git a/mysql-test/t/ctype_latin1.test b/mysql-test/t/ctype_latin1.test index 1ee48eed18c..3455b0437c1 100644 --- a/mysql-test/t/ctype_latin1.test +++ b/mysql-test/t/ctype_latin1.test @@ -389,6 +389,15 @@ EXPLAIN EXTENDED SELECT * FROM t1 WHERE COALESCE(c,0)='3 ' AND COALESCE(d,0)=COALESCE(c,0); DROP TABLE t1; +--echo # +--echo # MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple +--echo # + +SET NAMES latin1; +SELECT CONVERT(1, CHAR) IN ('100', 10, '101'); +SELECT CONVERT(1, CHAR) IN ('100', 10, '1'); +SELECT CONVERT(1, CHAR) IN ('100', '10', '1'); + --echo # --echo # End of 10.1 tests --echo # diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index e013109d0a7..be17eb461be 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -2024,6 +2024,16 @@ SELECT CONVERT(_utf8 0xC499 USING latin1); SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); +--echo # +--echo # MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple +--echo # + +SET NAMES utf8; +SELECT CONVERT(1, CHAR) IN ('100', 10, '101'); +SELECT CONVERT(1, CHAR) IN ('100', 10, '1'); +SELECT CONVERT(1, CHAR) IN ('100', '10', '1'); + + --echo # --echo # End of 10.1 tests --echo # diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 076e6da953c..41d5ce25fd4 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1312,6 +1312,13 @@ public: { value_res= item->val_str(&value); m_null_value= item->null_value; + // Make sure to cache the result String inside "value" + if (value_res && value_res != &value) + { + if (value.copy(*value_res)) + value.set("", 0, item->collation.collation); + value_res= &value; + } } int cmp(Item *arg) { From 7cf2428db31f0f3812e7c1b0923f61077717bb23 Mon Sep 17 00:00:00 2001 From: Jacob Mathew Date: Mon, 19 Mar 2018 12:49:22 -0700 Subject: [PATCH 10/19] MDEV-10991: Server crashes in spider_udf_direct_sql_create_conn - tests in spider/oracle* suites crash the server The crash occurs due to code that is #ifdef'd out with HAVE_ORACLE_OCI that pertains to the use of Spider with an Oracle data tier. Enabling this code eliminates the crash. The reason that MariaDB needs to support Oracle storage at the data tier is to help customers migrate from Oracle. It is necessary to build Spider with the additional build flag -DHAVE_ORACLE_OCI, and install and start Oracle before running the Oracle test suite or any tests within it. Nevertheless, if Spider is built normally and Oracle has not been started, these tests should not cause the MariaDB server to crash. The bug fix replaces the crash with the following error: ERROR 12501 (HY000) at line 4: The connect info 'ORACLE' is invalid Author: Jacob Mathew. Reviewer: Kentoku Shiba. --- storage/spider/spd_db_oracle.cc | 6 +++++- storage/spider/spd_direct_sql.cc | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/storage/spider/spd_db_oracle.cc b/storage/spider/spd_db_oracle.cc index c3dfe8b8cf2..8d7b9126570 100644 --- a/storage/spider/spd_db_oracle.cc +++ b/storage/spider/spd_db_oracle.cc @@ -29,6 +29,10 @@ #endif #ifdef HAVE_ORACLE_OCI +#if (defined(WIN32) || defined(_WIN32) || defined(WINDOWS) || defined(_WINDOWS)) +#include +#define strcasestr StrStr +#endif #include #include "spd_err.h" #include "spd_param.h" @@ -3817,7 +3821,7 @@ int spider_db_oracle_util::open_item_func( { Item_func_conv_charset *item_func_conv_charset = (Item_func_conv_charset *)item_func; - CHARSET_INFO *conv_charset = item_func_conv_charset->conv_charset; + CHARSET_INFO *conv_charset = item_func_conv_charset->collation.collation; uint cset_length = strlen(conv_charset->csname); if (str->reserve(SPIDER_SQL_USING_LEN + cset_length)) DBUG_RETURN(HA_ERR_OUT_OF_MEM); diff --git a/storage/spider/spd_direct_sql.cc b/storage/spider/spd_direct_sql.cc index 8f892869679..5c3ef9cabdf 100644 --- a/storage/spider/spd_direct_sql.cc +++ b/storage/spider/spd_direct_sql.cc @@ -371,6 +371,14 @@ SPIDER_CONN *spider_udf_direct_sql_create_conn( if (direct_sql->access_mode == 0) { #endif + if (direct_sql->dbton_id == SPIDER_DBTON_SIZE) + { + /* Invalid target wrapper */ + *error_num = ER_SPIDER_INVALID_CONNECT_INFO_NUM; + my_printf_error(*error_num, ER_SPIDER_INVALID_CONNECT_INFO_STR, + MYF(0), direct_sql->tgt_wrapper); + goto error_alloc_conn; + } if (!(conn = (SPIDER_CONN *) spider_bulk_malloc(spider_current_trx, 32, MYF(MY_WME | MY_ZEROFILL), &conn, sizeof(*conn), @@ -398,6 +406,14 @@ SPIDER_CONN *spider_udf_direct_sql_create_conn( conn->default_database.init_calc_mem(138); #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) } else { + if (direct_sql->dbton_id == SPIDER_DBTON_SIZE) + { + /* Invalid target wrapper */ + *error_num = ER_SPIDER_NOSQL_WRAPPER_IS_INVALID_NUM; + my_printf_error(*error_num, ER_SPIDER_NOSQL_WRAPPER_IS_INVALID_STR, + MYF(0), direct_sql->tgt_wrapper); + goto error_alloc_conn; + } if (!(conn = (SPIDER_CONN *) spider_bulk_malloc(spider_current_trx, 33, MYF(MY_WME | MY_ZEROFILL), &conn, sizeof(*conn), From e390f7b67509e324033644d017e8d6408d277d02 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 20 Mar 2018 11:05:28 +0530 Subject: [PATCH 11/19] MDEV-12737: tokudb_mariadb.mdev6657 fails in buildbot with different plan, and outside with valgrind warnings Fixing the test by adding replace column for the rows column in the explain. --- storage/tokudb/mysql-test/tokudb_mariadb/r/mdev6657.result | 6 +++--- storage/tokudb/mysql-test/tokudb_mariadb/t/mdev6657.test | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/mysql-test/tokudb_mariadb/r/mdev6657.result b/storage/tokudb/mysql-test/tokudb_mariadb/r/mdev6657.result index 2e9faddbaff..3804a583dc3 100644 --- a/storage/tokudb/mysql-test/tokudb_mariadb/r/mdev6657.result +++ b/storage/tokudb/mysql-test/tokudb_mariadb/r/mdev6657.result @@ -10,7 +10,7 @@ col3 smallint(5) NOT NULL DEFAULT '1', filler varchar(255) DEFAULT NULL, KEY pk_ersatz(col1,col2,col3), KEY key1 (col1,col2) USING BTREE -) ENGINE=TokuDB DEFAULT CHARSET=latin1 PACK_KEYS=1 COMPRESSION=TOKUDB_LZMA; +) ENGINE=TokuDB DEFAULT CHARSET=latin1 PACK_KEYS=1; insert into t3 select 1300000000+a, 12345, 7890, 'data' from t2; insert into t3 select 1400000000+a, 12345, 7890, 'data' from t2; insert into t3 select 1410799999+a, 12345, 7890, 'data' from t2; @@ -34,7 +34,7 @@ from t3 where col1 <= 1410799999 order by col1 desc,col2 desc,col3 desc limit 1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 range pk_ersatz,key1 pk_ersatz 4 NULL 2001 Using where; Using index +1 SIMPLE t3 range pk_ersatz,key1 pk_ersatz 4 NULL # Using where; Using index # The same query but the constant is bigger. # The query should use range(PRIMARY), not full index scan: explain @@ -43,5 +43,5 @@ from t3 where col1 <= 1412199999 order by col1 desc, col2 desc, col3 desc limit 1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 range pk_ersatz,key1 pk_ersatz 4 NULL 15001 Using where; Using index +1 SIMPLE t3 range pk_ersatz,key1 pk_ersatz 4 NULL # Using where; Using index drop table t1,t2,t3; diff --git a/storage/tokudb/mysql-test/tokudb_mariadb/t/mdev6657.test b/storage/tokudb/mysql-test/tokudb_mariadb/t/mdev6657.test index a809c3faf06..b723a5d72e8 100644 --- a/storage/tokudb/mysql-test/tokudb_mariadb/t/mdev6657.test +++ b/storage/tokudb/mysql-test/tokudb_mariadb/t/mdev6657.test @@ -15,7 +15,7 @@ CREATE TABLE t3 ( filler varchar(255) DEFAULT NULL, KEY pk_ersatz(col1,col2,col3), KEY key1 (col1,col2) USING BTREE -) ENGINE=TokuDB DEFAULT CHARSET=latin1 PACK_KEYS=1 COMPRESSION=TOKUDB_LZMA; +) ENGINE=TokuDB DEFAULT CHARSET=latin1 PACK_KEYS=1; insert into t3 select 1300000000+a, 12345, 7890, 'data' from t2; insert into t3 select 1400000000+a, 12345, 7890, 'data' from t2; @@ -35,6 +35,7 @@ insert into t3 select 1412099999+a, 12345, 7890, 'data' from t2; insert into t3 select 1412199999+a, 12345, 7890, 'data' from t2; --echo # The following must use range(PRIMARY): +--replace_column 9 # explain select col1,col2,col3 from t3 @@ -43,6 +44,7 @@ order by col1 desc,col2 desc,col3 desc limit 1; --echo # The same query but the constant is bigger. --echo # The query should use range(PRIMARY), not full index scan: +--replace_column 9 # explain select col1,col2,col3 from t3 From 2a729b5f4b14f9f04cf81e1d8dd4eec4ad6cb7cd Mon Sep 17 00:00:00 2001 From: sjaakola Date: Wed, 17 Feb 2016 11:20:48 +0200 Subject: [PATCH 12/19] refs MW-245 - merged wsrep_dirty_reads and wsrep_reject_queries from PXC --- .../suite/galera/r/galera_defaults.result | 3 +- .../galera/r/galera_schema_dirty_reads.result | 13 ++++++ .../r/galera_var_cluster_address.result | 3 +- .../galera/r/galera_var_dirty_reads.result | 7 ++-- .../galera/t/galera_schema_dirty_reads.test | 13 ++++++ .../galera/t/galera_var_cluster_address.test | 1 - .../galera/t/galera_var_dirty_reads.test | 3 -- .../suite/sys_vars/r/sysvars_wsrep.result | 14 +++++++ sql/sql_class.h | 8 ++++ sql/sql_parse.cc | 41 +++++++++++++++---- sql/sys_vars.cc | 10 ++++- sql/wsrep_mysqld.h | 7 ++++ sql/wsrep_var.cc | 25 +++++++++++ sql/wsrep_var.h | 1 + 14 files changed, 131 insertions(+), 18 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_schema_dirty_reads.result create mode 100644 mysql-test/suite/galera/t/galera_schema_dirty_reads.test diff --git a/mysql-test/suite/galera/r/galera_defaults.result b/mysql-test/suite/galera/r/galera_defaults.result index d3004735a0a..5c5fdabf432 100644 --- a/mysql-test/suite/galera/r/galera_defaults.result +++ b/mysql-test/suite/galera/r/galera_defaults.result @@ -1,6 +1,6 @@ SELECT COUNT(*) = 43 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_%'; COUNT(*) = 43 -1 +0 SELECT VARIABLE_NAME, VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_%' @@ -40,6 +40,7 @@ WSREP_NOTIFY_CMD WSREP_ON ON WSREP_OSU_METHOD TOI WSREP_RECOVER OFF +WSREP_REJECT_QUERIES NONE WSREP_REPLICATE_MYISAM OFF WSREP_RESTART_SLAVE OFF WSREP_RETRY_AUTOCOMMIT 1 diff --git a/mysql-test/suite/galera/r/galera_schema_dirty_reads.result b/mysql-test/suite/galera/r/galera_schema_dirty_reads.result new file mode 100644 index 00000000000..edf20da92c6 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_schema_dirty_reads.result @@ -0,0 +1,13 @@ +USE information_schema; +SELECT * FROM SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep_dirty_reads"; +VARIABLE_NAME VARIABLE_VALUE +WSREP_DIRTY_READS OFF +SET GLOBAL wsrep_reject_queries=ALL; +SELECT * FROM SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep_dirty_reads"; +VARIABLE_NAME VARIABLE_VALUE +WSREP_DIRTY_READS OFF +SET GLOBAL wsrep_reject_queries=NONE; +SET SESSION wsrep_dirty_reads=TRUE; +SELECT * FROM SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep_dirty_reads"; +VARIABLE_NAME VARIABLE_VALUE +WSREP_DIRTY_READS ON diff --git a/mysql-test/suite/galera/r/galera_var_cluster_address.result b/mysql-test/suite/galera/r/galera_var_cluster_address.result index 7c56e22eca2..8245cdf6093 100644 --- a/mysql-test/suite/galera/r/galera_var_cluster_address.result +++ b/mysql-test/suite/galera/r/galera_var_cluster_address.result @@ -1,7 +1,8 @@ SET GLOBAL wsrep_cluster_address = 'foo://'; SET SESSION wsrep_sync_wait=0; SELECT COUNT(*) > 0 FROM INFORMATION_SCHEMA.GLOBAL_STATUS; -ERROR 08S01: WSREP has not yet prepared node for application use +COUNT(*) > 0 +1 SHOW STATUS LIKE 'wsrep_ready'; Variable_name Value wsrep_ready OFF diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index f141b332b6c..5a108ddfcaa 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -33,11 +33,12 @@ SELECT 1; 1 1 USE information_schema; -ERROR 08S01: WSREP has not yet prepared node for application use SELECT * FROM information_schema.session_variables WHERE variable_name LIKE "wsrep_dirty_reads"; -ERROR 08S01: WSREP has not yet prepared node for application use +VARIABLE_NAME VARIABLE_VALUE +WSREP_DIRTY_READS OFF SELECT COUNT(*) >= 10 FROM performance_schema.events_statements_history; -ERROR 08S01: WSREP has not yet prepared node for application use +COUNT(*) >= 10 +1 USE test; SELECT * FROM t1; i diff --git a/mysql-test/suite/galera/t/galera_schema_dirty_reads.test b/mysql-test/suite/galera/t/galera_schema_dirty_reads.test new file mode 100644 index 00000000000..93e24244611 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_schema_dirty_reads.test @@ -0,0 +1,13 @@ +# +# Dirty reads from INFORMATION_SCHEMA tables. +# +--source include/galera_cluster.inc +--source include/have_innodb.inc +--disable_info +USE information_schema; +SELECT * FROM SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep_dirty_reads"; +SET GLOBAL wsrep_reject_queries=ALL; +SELECT * FROM SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep_dirty_reads"; +SET GLOBAL wsrep_reject_queries=NONE; +SET SESSION wsrep_dirty_reads=TRUE; +SELECT * FROM SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep_dirty_reads"; diff --git a/mysql-test/suite/galera/t/galera_var_cluster_address.test b/mysql-test/suite/galera/t/galera_var_cluster_address.test index 03706bbbb12..6d99d35cdac 100644 --- a/mysql-test/suite/galera/t/galera_var_cluster_address.test +++ b/mysql-test/suite/galera/t/galera_var_cluster_address.test @@ -24,7 +24,6 @@ SET GLOBAL wsrep_cluster_address = 'foo://'; SET SESSION wsrep_sync_wait=0; ---error ER_UNKNOWN_COM_ERROR SELECT COUNT(*) > 0 FROM INFORMATION_SCHEMA.GLOBAL_STATUS; # Must return 'OFF' diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index cba8488b879..138b7c1c703 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -49,13 +49,10 @@ SELECT i, variable_name, variable_value FROM t1, information_schema.session_vari SELECT 1; ---error ER_UNKNOWN_COM_ERROR USE information_schema; ---error ER_UNKNOWN_COM_ERROR SELECT * FROM information_schema.session_variables WHERE variable_name LIKE "wsrep_dirty_reads"; ---error ER_UNKNOWN_COM_ERROR SELECT COUNT(*) >= 10 FROM performance_schema.events_statements_history; --disable_query_log diff --git a/mysql-test/suite/sys_vars/r/sysvars_wsrep.result b/mysql-test/suite/sys_vars/r/sysvars_wsrep.result index 0c206975c29..db932ae8223 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_wsrep.result +++ b/mysql-test/suite/sys_vars/r/sysvars_wsrep.result @@ -421,6 +421,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY YES COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME WSREP_REJECT_QUERIES +SESSION_VALUE NULL +GLOBAL_VALUE NONE +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE NONE +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE ENUM +VARIABLE_COMMENT Variable to set to reject queries +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST NONE,ALL,ALL_KILL +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME WSREP_REPLICATE_MYISAM SESSION_VALUE NULL GLOBAL_VALUE OFF diff --git a/sql/sql_class.h b/sql/sql_class.h index 91030145022..b42b6ece9d4 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -5475,6 +5475,14 @@ public: sent by the user (ie: stored procedure). */ #define CF_SKIP_QUESTIONS (1U << 1) +#ifdef WITH_WSREP +/** + Do not check that wsrep snapshot is ready before allowing this command +*/ +#define CF_SKIP_WSREP_CHECK (1U << 2) +#else +#define CF_SKIP_WSREP_CHECK 0 +#endif /* WITH_WSREP */ /** Do not check that wsrep snapshot is ready before allowing this command diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 97d743d9a42..6a10f50813b 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2017, Oracle and/or its affiliates. - Copyright (c) 2008, 2017, MariaDB + Copyright (c) 2008, 2018, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -284,6 +284,8 @@ void init_update_queries(void) server_command_flags[COM_QUERY]= CF_SKIP_WSREP_CHECK; server_command_flags[COM_SET_OPTION]= CF_SKIP_WSREP_CHECK; server_command_flags[COM_STMT_PREPARE]= CF_SKIP_QUESTIONS | CF_SKIP_WSREP_CHECK; + server_command_flags[COM_STMT_EXECUTE]= CF_SKIP_WSREP_CHECK; + server_command_flags[COM_STMT_FETCH]= CF_SKIP_WSREP_CHECK; server_command_flags[COM_STMT_CLOSE]= CF_SKIP_QUESTIONS | CF_SKIP_WSREP_CHECK; server_command_flags[COM_STMT_RESET]= CF_SKIP_QUESTIONS | CF_SKIP_WSREP_CHECK; server_command_flags[COM_STMT_EXECUTE]= CF_SKIP_WSREP_CHECK; @@ -887,6 +889,25 @@ void cleanup_items(Item *item) } +#ifdef WITH_WSREP +static bool wsrep_tables_accessible_when_detached(const TABLE_LIST *tables) +{ + for (const TABLE_LIST *table= tables; table; table= table->next_global) + { + TABLE_CATEGORY c; + LEX_STRING db, tn; + lex_string_set(&db, table->db); + lex_string_set(&tn, table->table_name); + c= get_table_category(&db, &tn); + if (c != TABLE_CATEGORY_INFORMATION && + c != TABLE_CATEGORY_PERFORMANCE) + { + return false; + } + } + return true; +} +#endif /* WITH_WSREP */ #ifndef EMBEDDED_LIBRARY #ifdef WITH_WSREP @@ -2659,14 +2680,18 @@ mysql_execute_command(THD *thd) We additionally allow all other commands that do not change data in case wsrep_dirty_reads is enabled. */ - if (lex->sql_command != SQLCOM_SET_OPTION && - !wsrep_is_show_query(lex->sql_command) && - !(thd->variables.wsrep_dirty_reads && - !is_update_query(lex->sql_command)) && - !(lex->sql_command == SQLCOM_SELECT && - !all_tables) && - !wsrep_node_is_ready(thd)) + if (thd->variables.wsrep_on && !thd->wsrep_applier && + !(wsrep_ready || + (thd->variables.wsrep_dirty_reads && + (sql_command_flags[lex->sql_command] & CF_CHANGES_DATA) == 0) || + wsrep_tables_accessible_when_detached(all_tables)) && + lex->sql_command != SQLCOM_SET_OPTION && + !wsrep_is_show_query(lex->sql_command)) + { + my_message(ER_UNKNOWN_COM_ERROR, + "WSREP has not yet prepared node for application use", MYF(0)); goto error; + } } #endif /* WITH_WSREP */ status_var_increment(thd->status_var.com_stat[lex->sql_command]); diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 303633939c3..2ec886f0528 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2002, 2015, Oracle and/or its affiliates. - Copyright (c) 2012, 2015, MariaDB + Copyright (c) 2012, 2018, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -4967,6 +4967,14 @@ static Sys_var_mybool Sys_wsrep_desync ( ON_CHECK(wsrep_desync_check), ON_UPDATE(wsrep_desync_update)); +static const char *wsrep_reject_queries_names[]= { "NONE", "ALL", "ALL_KILL", NullS }; +static Sys_var_enum Sys_wsrep_reject_queries( + "wsrep_reject_queries", "Variable to set to reject queries", + GLOBAL_VAR(wsrep_reject_queries), CMD_LINE(OPT_ARG), + wsrep_reject_queries_names, DEFAULT(WSREP_REJECT_NONE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(wsrep_reject_queries_update)); + static const char *wsrep_binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", "NONE", NullS}; static Sys_var_enum Sys_wsrep_forced_binlog_format( diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index fd68fab991c..33af697a19b 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -80,6 +80,7 @@ extern const char* wsrep_notify_cmd; extern long wsrep_max_protocol_version; extern ulong wsrep_forced_binlog_format; extern my_bool wsrep_desync; +extern ulong wsrep_reject_queries; extern my_bool wsrep_replicate_myisam; extern ulong wsrep_mysql_replication_bundle; extern my_bool wsrep_restart_slave; @@ -92,6 +93,12 @@ extern bool wsrep_gtid_mode; extern uint32 wsrep_gtid_domain_id; extern bool wsrep_dirty_reads; +enum enum_wsrep_reject_types { + WSREP_REJECT_NONE, /* nothing rejected */ + WSREP_REJECT_ALL, /* reject all queries, with UNKNOWN_COMMAND error */ + WSREP_REJECT_ALL_KILL /* kill existing connections and reject all queries*/ +}; + enum enum_wsrep_OSU_method { WSREP_OSU_TOI, WSREP_OSU_RSU, diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index ad1f4ec0eac..da53c0981af 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -34,6 +34,7 @@ const char* wsrep_node_name = 0; const char* wsrep_node_address = 0; const char* wsrep_node_incoming_address = 0; const char* wsrep_start_position = 0; +ulong wsrep_reject_queries; static long wsrep_prev_slave_threads = wsrep_slave_threads; @@ -384,6 +385,30 @@ void wsrep_provider_options_init(const char* value) wsrep_provider_options = (value) ? my_strdup(value, MYF(0)) : NULL; } +bool wsrep_reject_queries_update(sys_var *self, THD* thd, enum_var_type type) +{ + switch (wsrep_reject_queries) { + case WSREP_REJECT_NONE: + wsrep_ready_set(TRUE); + WSREP_INFO("Allowing client queries due to manual setting"); + break; + case WSREP_REJECT_ALL: + wsrep_ready_set(FALSE); + WSREP_INFO("Rejecting client queries due to manual setting"); + break; + case WSREP_REJECT_ALL_KILL: + wsrep_ready_set(FALSE); + wsrep_close_client_connections(FALSE); + WSREP_INFO("Rejecting client queries and killing connections due to manual setting"); + break; + default: + WSREP_INFO("Unknown value for wsrep_reject_queries: %lu", + wsrep_reject_queries); + return true; + } + return false; +} + static int wsrep_cluster_address_verify (const char* cluster_address_str) { /* There is no predefined address format, it depends on provider. */ diff --git a/sql/wsrep_var.h b/sql/wsrep_var.h index 55eb2fbc501..53952173c83 100644 --- a/sql/wsrep_var.h +++ b/sql/wsrep_var.h @@ -93,6 +93,7 @@ extern bool wsrep_desync_update UPDATE_ARGS; extern bool wsrep_max_ws_size_check CHECK_ARGS; extern bool wsrep_max_ws_size_update UPDATE_ARGS; +extern bool wsrep_reject_queries_update UPDATE_ARGS; #else /* WITH_WSREP */ From 8f717ed36002e6257b0ba659863f3a48d5c1f579 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 19 Feb 2016 00:14:26 +0200 Subject: [PATCH 13/19] refs MW-245 - allowing USE with dirty reads configuration - fix for logic of setting wsrep ready status --- sql/sql_parse.cc | 34 ++++++++++++++++++---------------- sql/wsrep_var.cc | 3 --- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 6a10f50813b..c6adfd0999a 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -275,6 +275,7 @@ void init_update_queries(void) server_command_flags[COM_SHUTDOWN]= CF_SKIP_WSREP_CHECK; server_command_flags[COM_SLEEP]= CF_SKIP_WSREP_CHECK; server_command_flags[COM_TIME]= CF_SKIP_WSREP_CHECK; + server_command_flags[COM_INIT_DB]= CF_SKIP_WSREP_CHECK; server_command_flags[COM_END]= CF_SKIP_WSREP_CHECK; /* @@ -913,7 +914,8 @@ static bool wsrep_tables_accessible_when_detached(const TABLE_LIST *tables) #ifdef WITH_WSREP static bool wsrep_node_is_ready(THD *thd) { - if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready) + if (thd->variables.wsrep_on && !thd->wsrep_applier && + (!wsrep_ready || wsrep_reject_queries != WSREP_REJECT_NONE)) { my_message(ER_UNKNOWN_COM_ERROR, "WSREP has not yet prepared node for application use", @@ -1104,8 +1106,9 @@ bool do_command(THD *thd) /* Bail out if DB snapshot has not been installed. */ - if (!(server_command_flags[command] & CF_SKIP_WSREP_CHECK) && - !wsrep_node_is_ready(thd)) + if (thd->variables.wsrep_on && !thd->wsrep_applier && + (!wsrep_ready || wsrep_reject_queries != WSREP_REJECT_NONE) && + (server_command_flags[command] & CF_SKIP_WSREP_CHECK) == 0) { thd->protocol->end_statement(); @@ -2674,19 +2677,18 @@ mysql_execute_command(THD *thd) } /* - Bail out if DB snapshot has not been installed. SET and SHOW commands, - however, are always allowed. - Select query is also allowed if it does not access any table. - We additionally allow all other commands that do not change data in - case wsrep_dirty_reads is enabled. - */ - if (thd->variables.wsrep_on && !thd->wsrep_applier && - !(wsrep_ready || - (thd->variables.wsrep_dirty_reads && - (sql_command_flags[lex->sql_command] & CF_CHANGES_DATA) == 0) || - wsrep_tables_accessible_when_detached(all_tables)) && - lex->sql_command != SQLCOM_SET_OPTION && - !wsrep_is_show_query(lex->sql_command)) + * Bail out if DB snapshot has not been installed. We however, + * allow SET and SHOW queries and reads from information schema + * and dirty reads (if configured) + */ + if (thd->variables.wsrep_on && + !thd->wsrep_applier && + !(wsrep_ready && wsrep_reject_queries == WSREP_REJECT_NONE) && + !(thd->variables.wsrep_dirty_reads && + (sql_command_flags[lex->sql_command] & CF_CHANGES_DATA) == 0) && + !wsrep_tables_accessible_when_detached(all_tables) && + lex->sql_command != SQLCOM_SET_OPTION && + !wsrep_is_show_query(lex->sql_command)) { my_message(ER_UNKNOWN_COM_ERROR, "WSREP has not yet prepared node for application use", MYF(0)); diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index da53c0981af..2d52396331d 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -389,15 +389,12 @@ bool wsrep_reject_queries_update(sys_var *self, THD* thd, enum_var_type type) { switch (wsrep_reject_queries) { case WSREP_REJECT_NONE: - wsrep_ready_set(TRUE); WSREP_INFO("Allowing client queries due to manual setting"); break; case WSREP_REJECT_ALL: - wsrep_ready_set(FALSE); WSREP_INFO("Rejecting client queries due to manual setting"); break; case WSREP_REJECT_ALL_KILL: - wsrep_ready_set(FALSE); wsrep_close_client_connections(FALSE); WSREP_INFO("Rejecting client queries and killing connections due to manual setting"); break; From 84d4ab5be1d6490742353c9c7c7ae198ceb66b42 Mon Sep 17 00:00:00 2001 From: Philip Stoev Date: Fri, 19 Feb 2016 00:16:36 -0800 Subject: [PATCH 14/19] refs MW-245: Galera MTR Tests: additional tests for wsrep_reject_queries, wsrep_dirty_reads --- .../galera/r/galera_var_reject_queries.result | 21 +++++++++ .../galera/t/galera_var_reject_queries.test | 44 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 mysql-test/suite/galera/r/galera_var_reject_queries.result create mode 100644 mysql-test/suite/galera/t/galera_var_reject_queries.test diff --git a/mysql-test/suite/galera/r/galera_var_reject_queries.result b/mysql-test/suite/galera/r/galera_var_reject_queries.result new file mode 100644 index 00000000000..e4b906f1717 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_var_reject_queries.result @@ -0,0 +1,21 @@ +CREATE TABLE t1 (f1 INTEGER); +SET SESSION wsrep_reject_queries = ALL; +ERROR HY000: Variable 'wsrep_reject_queries' is a GLOBAL variable and should be set with SET GLOBAL +SET GLOBAL wsrep_reject_queries = ALL; +SELECT * FROM t1; +ERROR 08S01: WSREP has not yet prepared node for application use +SET GLOBAL wsrep_reject_queries = ALL_KILL; +ERROR HY000: Lost connection to MySQL server during query +SELECT * FROM t1; +ERROR 70100: Connection was killed +SELECT * FROM t1; +ERROR 08S01: WSREP has not yet prepared node for application use +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +VARIABLE_VALUE = 2 +1 +INSERT INTO t1 VALUES (1); +SET GLOBAL wsrep_reject_queries = NONE; +SELECT COUNT(*) = 1 FROM t1; +COUNT(*) = 1 +1 +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_var_reject_queries.test b/mysql-test/suite/galera/t/galera_var_reject_queries.test new file mode 100644 index 00000000000..b89cae78aa6 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_var_reject_queries.test @@ -0,0 +1,44 @@ +# +# Test wsrep_reject_queries +# + +--source include/galera_cluster.inc +--source include/have_innodb.inc + +CREATE TABLE t1 (f1 INTEGER); + +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 + +--connection node_1 +--error ER_GLOBAL_VARIABLE +SET SESSION wsrep_reject_queries = ALL; + +SET GLOBAL wsrep_reject_queries = ALL; + +--error ER_UNKNOWN_COM_ERROR +SELECT * FROM t1; + +# Lost connection +--error 2013 +SET GLOBAL wsrep_reject_queries = ALL_KILL; + +--connection node_1a +--error ER_CONNECTION_KILLED +SELECT * FROM t1; + +--connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1 +--error ER_UNKNOWN_COM_ERROR +SELECT * FROM t1; + +# Confirm that replication continues + +--connection node_2 +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +INSERT INTO t1 VALUES (1); + +--connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1 +SET GLOBAL wsrep_reject_queries = NONE; + +SELECT COUNT(*) = 1 FROM t1; + +DROP TABLE t1; From c5dd2abf4ca26cb4ac6bfc3123500e5c2f46cbff Mon Sep 17 00:00:00 2001 From: sjaakola Date: Wed, 2 Mar 2016 21:32:06 +0200 Subject: [PATCH 15/19] Refs MW-245 - logic was wrong in detecting if queries are allowed in non primary node. it allowed select with no table list to execute even if dirty reads was not specified --- sql/sql_parse.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index c6adfd0999a..c1962508864 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2686,7 +2686,8 @@ mysql_execute_command(THD *thd) !(wsrep_ready && wsrep_reject_queries == WSREP_REJECT_NONE) && !(thd->variables.wsrep_dirty_reads && (sql_command_flags[lex->sql_command] & CF_CHANGES_DATA) == 0) && - !wsrep_tables_accessible_when_detached(all_tables) && + !(thd->variables.wsrep_dirty_reads && + wsrep_tables_accessible_when_detached(all_tables)) && lex->sql_command != SQLCOM_SET_OPTION && !wsrep_is_show_query(lex->sql_command)) { From 9a8961485763c057620a24e60713d15d22ab41b1 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 3 Mar 2016 09:35:52 +0200 Subject: [PATCH 16/19] Refs: MW-245 - changed logic so that in non primary node it is possible to do SET + SHOW + SELECT from information and pfs schema, when dirty reads are not enabled - however, non table selects are not allowed (e.g. SELECT 1) --- sql/sql_parse.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index c1962508864..affadf01d2c 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -893,6 +893,7 @@ void cleanup_items(Item *item) #ifdef WITH_WSREP static bool wsrep_tables_accessible_when_detached(const TABLE_LIST *tables) { + bool has_tables = false; for (const TABLE_LIST *table= tables; table; table= table->next_global) { TABLE_CATEGORY c; @@ -905,8 +906,9 @@ static bool wsrep_tables_accessible_when_detached(const TABLE_LIST *tables) { return false; } + has_tables = true; } - return true; + return has_tables; } #endif /* WITH_WSREP */ #ifndef EMBEDDED_LIBRARY @@ -2686,8 +2688,7 @@ mysql_execute_command(THD *thd) !(wsrep_ready && wsrep_reject_queries == WSREP_REJECT_NONE) && !(thd->variables.wsrep_dirty_reads && (sql_command_flags[lex->sql_command] & CF_CHANGES_DATA) == 0) && - !(thd->variables.wsrep_dirty_reads && - wsrep_tables_accessible_when_detached(all_tables)) && + !wsrep_tables_accessible_when_detached(all_tables) && lex->sql_command != SQLCOM_SET_OPTION && !wsrep_is_show_query(lex->sql_command)) { From 33028f7c4bc4b702a9dd2d8462f2f79b8f37dc1f Mon Sep 17 00:00:00 2001 From: Philip Stoev Date: Thu, 3 Mar 2016 02:57:21 -0800 Subject: [PATCH 17/19] Refs: MW-245 - Adjust tests to account for the new behavior. --- mysql-test/suite/galera/r/galera_var_dirty_reads.result | 4 ++-- mysql-test/suite/galera/r/galera_var_reject_queries.result | 2 +- mysql-test/suite/galera/t/galera_as_slave_nonprim.test | 7 ++++--- mysql-test/suite/galera/t/galera_var_dirty_reads.test | 2 ++ mysql-test/suite/galera/t/galera_var_reject_queries.test | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 5a108ddfcaa..da842e7254a 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -30,9 +30,9 @@ SET @@session.wsrep_dirty_reads=OFF; SELECT i, variable_name, variable_value FROM t1, information_schema.session_variables WHERE variable_name LIKE "wsrep_dirty_reads" AND i = 1; ERROR 08S01: WSREP has not yet prepared node for application use SELECT 1; -1 -1 +ERROR 08S01: WSREP has not yet prepared node for application use USE information_schema; +ERROR 08S01: WSREP has not yet prepared node for application use SELECT * FROM information_schema.session_variables WHERE variable_name LIKE "wsrep_dirty_reads"; VARIABLE_NAME VARIABLE_VALUE WSREP_DIRTY_READS OFF diff --git a/mysql-test/suite/galera/r/galera_var_reject_queries.result b/mysql-test/suite/galera/r/galera_var_reject_queries.result index e4b906f1717..5958e6ae981 100644 --- a/mysql-test/suite/galera/r/galera_var_reject_queries.result +++ b/mysql-test/suite/galera/r/galera_var_reject_queries.result @@ -7,7 +7,7 @@ ERROR 08S01: WSREP has not yet prepared node for application use SET GLOBAL wsrep_reject_queries = ALL_KILL; ERROR HY000: Lost connection to MySQL server during query SELECT * FROM t1; -ERROR 70100: Connection was killed +Got one of the listed errors SELECT * FROM t1; ERROR 08S01: WSREP has not yet prepared node for application use SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; diff --git a/mysql-test/suite/galera/t/galera_as_slave_nonprim.test b/mysql-test/suite/galera/t/galera_as_slave_nonprim.test index 46a93458271..26fec05dfe5 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_nonprim.test +++ b/mysql-test/suite/galera/t/galera_as_slave_nonprim.test @@ -51,7 +51,7 @@ INSERT INTO t1 VALUES (1),(2),(3),(4),(5); --let $value = query_get_value(SHOW SLAVE STATUS, Last_SQL_Error, 1) --connection node_3 --disable_query_log ---eval SELECT "$value" IN ("Error 'Unknown command' on query. Default database: 'test'. Query: 'BEGIN'", "Node has dropped from cluster") AS expected_error +--eval SELECT "$value" IN ("Error 'WSREP has not yet prepared node for application use' on query. Default database: 'test'. Query: 'BEGIN'", "Node has dropped from cluster") AS expected_error --enable_query_log # Step #4. Bring back the async slave and restart replication @@ -85,8 +85,9 @@ DROP TABLE t1; STOP SLAVE; RESET SLAVE ALL; -CALL mtr.add_suppression("Slave SQL: Error 'Unknown command' on query"); -CALL mtr.add_suppression("Slave: Unknown command Error_code: 1047"); +CALL mtr.add_suppression("Slave SQL: Error 'WSREP has not yet prepared node for application use' on query"); +CALL mtr.add_suppression("Slave: WSREP has not yet prepared node for application use Error_code: 1047"); +CALL mtr.add_suppression("TORDERED} returned -107 \\(Transport endpoint is not connected\\)"); CALL mtr.add_suppression("Transport endpoint is not connected"); CALL mtr.add_suppression("Slave SQL: Error in Xid_log_event: Commit could not be completed, 'Deadlock found when trying to get lock; try restarting transaction', Error_code: 1213"); CALL mtr.add_suppression("Slave SQL: Node has dropped from cluster, Error_code: 1047"); diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index 138b7c1c703..85d759e4a27 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -47,8 +47,10 @@ SET @@session.wsrep_dirty_reads=OFF; --error ER_UNKNOWN_COM_ERROR SELECT i, variable_name, variable_value FROM t1, information_schema.session_variables WHERE variable_name LIKE "wsrep_dirty_reads" AND i = 1; +--error ER_UNKNOWN_COM_ERROR SELECT 1; +--error ER_UNKNOWN_COM_ERROR USE information_schema; SELECT * FROM information_schema.session_variables WHERE variable_name LIKE "wsrep_dirty_reads"; diff --git a/mysql-test/suite/galera/t/galera_var_reject_queries.test b/mysql-test/suite/galera/t/galera_var_reject_queries.test index b89cae78aa6..b1af9d8aa2b 100644 --- a/mysql-test/suite/galera/t/galera_var_reject_queries.test +++ b/mysql-test/suite/galera/t/galera_var_reject_queries.test @@ -23,7 +23,7 @@ SELECT * FROM t1; SET GLOBAL wsrep_reject_queries = ALL_KILL; --connection node_1a ---error ER_CONNECTION_KILLED +--error ER_CONNECTION_KILLED,2013 SELECT * FROM t1; --connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1 From bc2e7d7889e35f30390d1ef8653f6ac9c038b5b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 20 Mar 2018 12:10:17 +0200 Subject: [PATCH 18/19] Fix test case MW-329. --- mysql-test/suite/galera/r/MW-329.result | 1 + mysql-test/suite/galera/t/MW-329.test | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mysql-test/suite/galera/r/MW-329.result b/mysql-test/suite/galera/r/MW-329.result index a79ba598d69..4666d131c7d 100644 --- a/mysql-test/suite/galera/r/MW-329.result +++ b/mysql-test/suite/galera/r/MW-329.result @@ -19,3 +19,4 @@ VARIABLE_VALUE > 0 DROP PROCEDURE proc_insert; DROP TABLE t1; CALL mtr.add_suppression("conflict state 3 after post commit"); +set global innodb_status_output=Default; diff --git a/mysql-test/suite/galera/t/MW-329.test b/mysql-test/suite/galera/t/MW-329.test index d9f9a787442..bf045832113 100644 --- a/mysql-test/suite/galera/t/MW-329.test +++ b/mysql-test/suite/galera/t/MW-329.test @@ -83,3 +83,5 @@ DROP TABLE t1; # Due to MW-330, Multiple "conflict state 3 after post commit" warnings if table is dropped while SP is running CALL mtr.add_suppression("conflict state 3 after post commit"); + +set global innodb_status_output=Default; \ No newline at end of file From eee73ddfbb29816320c9fc78c8ff1012cac6567a Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 20 Mar 2018 17:51:57 +0530 Subject: [PATCH 19/19] MDEV-12255 innodb_prefix_index_cluster_optimization hits debug build assert on UTF-8 columns Problem: ======= (1) Multi-byte character cases are not considered during prefix index cluster optimization check. It leads to fetch of improper results during read operation. (2) Strict assert in row_sel_field_store_in_mysql_format_func and it asserts for prefix index record to mysql conversion. Solution: ======== (1) Consider the case of multi-byte character during prefix index cluster optimization check. (2) Relax the assert in row_sel_field_store_in_mysql_format_func to allow prefix index record to mysql format conversion. The patch is taken from https://github.com/laurynas-biveinis/percona-server/commit/1eee538087ffcf121c37f844b447ba5480faf081 --- .../r/fast_prefix_index_fetch_innodb.result | 361 +++++++++- .../t/fast_prefix_index_fetch_innodb.test | 664 ++++++++++++++++-- storage/innobase/row/row0sel.cc | 183 +++-- storage/xtradb/row/row0sel.cc | 180 +++-- 4 files changed, 1155 insertions(+), 233 deletions(-) diff --git a/mysql-test/r/fast_prefix_index_fetch_innodb.result b/mysql-test/r/fast_prefix_index_fetch_innodb.result index 92af85f7fdb..c6d96389b08 100644 --- a/mysql-test/r/fast_prefix_index_fetch_innodb.result +++ b/mysql-test/r/fast_prefix_index_fetch_innodb.result @@ -30,73 +30,372 @@ id fake_id bigfield 33 1033 yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy 128 1128 zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz # Baseline sanity check: 0, 0. +select "no-op query"; no-op query no-op query -cluster_lookups_matched -1 -cluster_lookups_avoided_matched -1 +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 # Eligible for optimization. +select id, bigfield from prefixinno where bigfield = repeat('d', 31); id bigfield 31 ddddddddddddddddddddddddddddddd -cluster_lookups_matched -1 -cluster_lookups_avoided_matched +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided 1 # Eligible for optimization, access via fake_id only. +select id, bigfield from prefixinno where fake_id = 1031; id bigfield 31 ddddddddddddddddddddddddddddddd -cluster_lookups_matched -1 -cluster_lookups_avoided_matched +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided 1 # Not eligible for optimization, access via fake_id of big row. +select id, bigfield from prefixinno where fake_id = 1033; id bigfield 33 yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy -cluster_lookups_matched -1 -cluster_lookups_avoided_matched +select @cluster_lookups; +@cluster_lookups 1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 # Not eligible for optimization. +select id, bigfield from prefixinno where bigfield = repeat('x', 32); id bigfield 32 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -cluster_lookups_matched -1 -cluster_lookups_avoided_matched +select @cluster_lookups; +@cluster_lookups 1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 # Not eligible for optimization. +select id, bigfield from prefixinno where bigfield = repeat('y', 33); id bigfield 33 yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy -cluster_lookups_matched -1 -cluster_lookups_avoided_matched +select @cluster_lookups; +@cluster_lookups 1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 # Eligible, should not increment lookup counter. +select id, bigfield from prefixinno where bigfield = repeat('b', 8); id bigfield 8 bbbbbbbb -cluster_lookups_matched -1 -cluster_lookups_avoided_matched +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided 1 # Eligible, should not increment lookup counter. +select id, bigfield from prefixinno where bigfield = repeat('c', 24); id bigfield 24 cccccccccccccccccccccccc -cluster_lookups_matched -1 -cluster_lookups_avoided_matched +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided 1 # Should increment lookup counter. +select id, bigfield from prefixinno where bigfield = repeat('z', 128); id bigfield 128 zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz -cluster_lookups_matched -1 -cluster_lookups_avoided_matched +select @cluster_lookups; +@cluster_lookups 1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 # Disable optimization, confirm we still increment counter. +set global innodb_prefix_index_cluster_optimization = OFF; +select id, bigfield from prefixinno where fake_id = 1033; id bigfield 33 yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy -cluster_lookups_matched +select @cluster_lookups; +@cluster_lookups 1 -cluster_lookups_avoided_matched +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +drop table prefixinno; +# Multi-byte handling case +set global innodb_prefix_index_cluster_optimization = ON; +SET NAMES utf8mb4; +CREATE TABLE t1( +f1 varchar(10) CHARACTER SET UTF8MB4 COLLATE UTF8MB4_BIN, +INDEX (f1(3)))ENGINE=INNODB; +INSERT INTO t1 VALUES('a'), ('cccc'), ('až'), ('cčc'), ('ggᵷg'), ('¢¢'); +INSERT INTO t1 VALUES('தமிழ்'), ('🐱🌑'), ('🌒'), ('🌑'); +INSERT INTO t1 VALUES('😊me'), ('eu€'), ('ls¢'); +# Eligible - record length is shorter than prefix +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'a'; +f1 +a +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided 1 -# make test suite happy by cleaning up our mess +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like 'c%'; +f1 +cccc +cčc +select @cluster_lookups; +@cluster_lookups +3 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Eligible - record length shorter than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'až'; +f1 +až +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +1 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'தமிழ்'; +f1 +தமிழ் +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like 'ggᵷ%'; +f1 +ggᵷg +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '😊%'; +f1 +😊me +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'ls¢'; +f1 +ls¢ +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Eligible - record length shorter than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '¢¢%'; +f1 +¢¢ +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +1 +# Eligible - record length shorter than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🐱🌑%'; +f1 +🐱🌑 +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +1 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌑%'; +f1 +🌑 +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +2 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌒%'; +f1 +🌒 +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +2 +DROP TABLE t1; +# Multi-byte with minimum character length > 1 bytes +CREATE TABLE t1( +f1 varchar(10) CHARACTER SET UTF16 COLLATE UTF16_BIN, +INDEX (f1(3)))ENGINE=INNODB; +INSERT INTO t1 VALUES('a'), ('cccc'), ('až'), ('cčc'), ('ggᵷg'), ('¢¢'); +INSERT INTO t1 VALUES('தமிழ்'), ('🐱🌑'), ('🌒'), ('🌑'); +INSERT INTO t1 VALUES('😊me'), ('eu€'), ('ls¢'); +# Eligible - record length is shorter than prefix +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'a'; +f1 +a +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +1 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like 'c%'; +f1 +cccc +cčc +select @cluster_lookups; +@cluster_lookups +3 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Eligible - record length shorter than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'až'; +f1 +až +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +1 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'தமிழ்'; +f1 +தமிழ் +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like 'ggᵷ%'; +f1 +ggᵷg +select @cluster_lookups; +@cluster_lookups +2 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '😊%'; +f1 +😊me +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Not eligible - record length longer than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'ls¢'; +f1 +ls¢ +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Eligible - record length shorter than prefix length +SELECT f1 FROM t1 FORCE INDEX(`f1`) WHERE f1 like '¢¢%'; +f1 +¢¢ +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +1 +# Eligible - record length shorter than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🐱🌑%'; +f1 +🐱🌑 +select @cluster_lookups; +@cluster_lookups +2 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +# Eligible - record length is shorter than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌑%'; +f1 +🌑 +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +2 +# Eligible - record length is shorter than prefix length +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌒%'; +f1 +🌒 +select @cluster_lookups; +@cluster_lookups +1 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +1 +DROP TABLE t1; +CREATE TABLE t1( +col1 INT, +col2 BLOB DEFAULT NULL, +INDEX `idx1`(col2(4), col1))ENGINE=INNODB; +INSERT INTO t1 VALUES (2, 'test'), (3, repeat('test1', 2000)); +INSERT INTO t1(col1) VALUES(1); +# Eligible - record length is shorter than prefix length +SELECT col1 FROM t1 FORCE INDEX (`idx1`) WHERE col2 is NULL; +col1 +1 +select @cluster_lookups; +@cluster_lookups +0 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +1 +# Not eligible - record length longer than prefix index +SELECT col1 FROM t1 FORCE INDEX (`idx1`) WHERE col2 like 'test1%'; +col1 +3 +select @cluster_lookups; +@cluster_lookups +2 +select @cluster_lookups_avoided; +@cluster_lookups_avoided +0 +DROP TABLE t1; +set global innodb_prefix_index_cluster_optimization = OFF; diff --git a/mysql-test/t/fast_prefix_index_fetch_innodb.test b/mysql-test/t/fast_prefix_index_fetch_innodb.test index e563e65ec2a..c3b3440d82d 100644 --- a/mysql-test/t/fast_prefix_index_fetch_innodb.test +++ b/mysql-test/t/fast_prefix_index_fetch_innodb.test @@ -31,120 +31,638 @@ select * from prefixinno; let $show_count_statement = show status like 'innodb_secondary_index_triggered_cluster_reads'; let $show_opt_statement = show status like 'innodb_secondary_index_triggered_cluster_reads_avoided'; ---disable_query_log - --echo # Baseline sanity check: 0, 0. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select "no-op query"; ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Eligible for optimization. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select id, bigfield from prefixinno where bigfield = repeat('d', 31); ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Eligible for optimization, access via fake_id only. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select id, bigfield from prefixinno where fake_id = 1031; ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Not eligible for optimization, access via fake_id of big row. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select id, bigfield from prefixinno where fake_id = 1033; ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Not eligible for optimization. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select id, bigfield from prefixinno where bigfield = repeat('x', 32); ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Not eligible for optimization. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select id, bigfield from prefixinno where bigfield = repeat('y', 33); ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Eligible, should not increment lookup counter. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select id, bigfield from prefixinno where bigfield = repeat('b', 8); ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Eligible, should not increment lookup counter. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select id, bigfield from prefixinno where bigfield = repeat('c', 24); ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Should increment lookup counter. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + select id, bigfield from prefixinno where bigfield = repeat('z', 128); ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; --echo # Disable optimization, confirm we still increment counter. ---let $base_count = query_get_value($show_count_statement, Value, 1) ---let $base_opt = query_get_value($show_opt_statement, Value, 1) +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + set global innodb_prefix_index_cluster_optimization = OFF; select id, bigfield from prefixinno where fake_id = 1033; ---let $count = query_get_value($show_count_statement, Value, 1) + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log eval select $count - $base_count into @cluster_lookups; -select @cluster_lookups = 1 as cluster_lookups_matched; ---let $opt = query_get_value($show_opt_statement, Value, 1) -eval select $opt - $base_opt into @cluster_lookups; -select @cluster_lookups = 0 as cluster_lookups_avoided_matched; +eval select $opt - $base_opt into @cluster_lookups_avoided; +--enable_query_log +select @cluster_lookups; +select @cluster_lookups_avoided; ---echo # make test suite happy by cleaning up our mess drop table prefixinno; + +--echo # Multi-byte handling case + +set global innodb_prefix_index_cluster_optimization = ON; +SET NAMES utf8mb4; +CREATE TABLE t1( + f1 varchar(10) CHARACTER SET UTF8MB4 COLLATE UTF8MB4_BIN, + INDEX (f1(3)))ENGINE=INNODB; + +INSERT INTO t1 VALUES('a'), ('cccc'), ('až'), ('cčc'), ('ggᵷg'), ('¢¢'); +INSERT INTO t1 VALUES('தமிழ்'), ('🐱🌑'), ('🌒'), ('🌑'); +INSERT INTO t1 VALUES('😊me'), ('eu€'), ('ls¢'); + +--echo # Eligible - record length is shorter than prefix +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'a'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like 'c%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Eligible - record length shorter than prefix length +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'až'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'தமிழ்'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like 'ggᵷ%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '😊%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'ls¢'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Eligible - record length shorter than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '¢¢%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Eligible - record length shorter than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🐱🌑%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌑%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌒%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +DROP TABLE t1; + +--echo # Multi-byte with minimum character length > 1 bytes + +CREATE TABLE t1( + f1 varchar(10) CHARACTER SET UTF16 COLLATE UTF16_BIN, + INDEX (f1(3)))ENGINE=INNODB; + +INSERT INTO t1 VALUES('a'), ('cccc'), ('až'), ('cčc'), ('ggᵷg'), ('¢¢'); +INSERT INTO t1 VALUES('தமிழ்'), ('🐱🌑'), ('🌒'), ('🌑'); +INSERT INTO t1 VALUES('😊me'), ('eu€'), ('ls¢'); + +--echo # Eligible - record length is shorter than prefix +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'a'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like 'c%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Eligible - record length shorter than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'až'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'தமிழ்'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like 'ggᵷ%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '😊%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 = 'ls¢'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Eligible - record length shorter than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX(`f1`) WHERE f1 like '¢¢%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Eligible - record length shorter than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🐱🌑%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Eligible - record length is shorter than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌑%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Eligible - record length is shorter than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌒%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +DROP TABLE t1; + +CREATE TABLE t1( + col1 INT, + col2 BLOB DEFAULT NULL, + INDEX `idx1`(col2(4), col1))ENGINE=INNODB; +INSERT INTO t1 VALUES (2, 'test'), (3, repeat('test1', 2000)); +INSERT INTO t1(col1) VALUES(1); + +--echo # Eligible - record length is shorter than prefix length + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT col1 FROM t1 FORCE INDEX (`idx1`) WHERE col2 is NULL; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +--echo # Not eligible - record length longer than prefix index + +let $base_count = query_get_value($show_count_statement, Value, 1); +let $base_opt = query_get_value($show_opt_statement, Value, 1); + +SELECT col1 FROM t1 FORCE INDEX (`idx1`) WHERE col2 like 'test1%'; + +let $count = query_get_value($show_count_statement, Value, 1); +let $opt = query_get_value($show_opt_statement, Value, 1); + +--disable_query_log +eval set @cluster_lookups = $count - $base_count; +eval set @cluster_lookups_avoided = $opt - $base_opt; +--enable_query_log + +select @cluster_lookups; +select @cluster_lookups_avoided; + +DROP TABLE t1; set global innodb_prefix_index_cluster_optimization = OFF; diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 3cf7bc6ee80..7af788973f2 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -2707,7 +2707,9 @@ row_sel_field_store_in_mysql_format_func( || !(templ->mysql_col_len % templ->mbmaxlen)); ut_ad(len * templ->mbmaxlen >= templ->mysql_col_len || (field_no == templ->icp_rec_field_no - && field->prefix_len > 0)); + && field->prefix_len > 0) + || templ->rec_field_is_prefix); + ut_ad(!(field->prefix_len % templ->mbmaxlen)); if (templ->mbminlen == 1 && templ->mbmaxlen != 1) { @@ -3667,6 +3669,118 @@ row_search_idx_cond_check( return(result); } +/** Return the record field length in characters. +@param[in] col table column of the field +@param[in] field_no field number +@param[in] rec physical record +@param[in] offsets field offsets in the physical record +@return field length in characters. */ +static +size_t +rec_field_len_in_chars( + const dict_col_t* col, + const ulint field_no, + const rec_t* rec, + const ulint* offsets) +{ + const ulint cset = dtype_get_charset_coll(col->prtype); + const CHARSET_INFO* cs = all_charsets[cset]; + ulint rec_field_len; + const char* rec_field = reinterpret_cast( + rec_get_nth_field( + rec, offsets, field_no, &rec_field_len)); + + if (UNIV_UNLIKELY(!cs)) { + ib_logf(IB_LOG_LEVEL_WARN, "Missing collation " ULINTPF, cset); + return SIZE_T_MAX; + } + + return(cs->cset->numchars(cs, rec_field, rec_field + rec_field_len)); +} + +/** Avoid the clustered index lookup if all the following conditions +are true: +1) all columns are in secondary index +2) all values for columns that are prefix-only indexes are shorter +than the prefix size. This optimization can avoid many IOs for certain schemas. +@return true, to avoid clustered index lookup. */ +static +bool row_search_with_covering_prefix( + row_prebuilt_t* prebuilt, + const rec_t* rec, + const ulint* offsets) +{ + const dict_index_t* index = prebuilt->index; + ut_ad(!dict_index_is_clust(index)); + + if (!srv_prefix_index_cluster_optimization) { + return false; + } + + /** Optimization only applicable if there the number of secondary index + fields are greater than or equal to number of clustered index fields. */ + if (prebuilt->n_template > index->n_fields) { + return false; + } + + for (ulint i = 0; i < prebuilt->n_template; i++) { + mysql_row_templ_t* templ = prebuilt->mysql_template + i; + ulint j = templ->rec_prefix_field_no; + + /** Condition (1) : is the field in the index. */ + if (j == ULINT_UNDEFINED) { + return false; + } + + /** Condition (2): If this is a prefix index then + row's value size shorter than prefix length. */ + + if (!templ->rec_field_is_prefix) { + continue; + } + + ulint rec_size = rec_offs_nth_size(offsets, j); + const dict_field_t* field = dict_index_get_nth_field(index, j); + ulint max_chars = field->prefix_len / templ->mbmaxlen; + + ut_a(field->prefix_len > 0); + + if (rec_size < max_chars) { + /* Record in bytes shorter than the index + prefix length in char. */ + continue; + } + + if (rec_size * templ->mbminlen >= field->prefix_len) { + /* Shortest representation string by the + byte length of the record is longer than the + maximum possible index prefix. */ + return false; + } + + size_t num_chars = rec_field_len_in_chars( + field->col, j, rec, offsets); + + if (num_chars >= max_chars) { + /* No of chars to store the record exceeds + the index prefix character length. */ + return false; + } + } + + /* If prefix index optimization condition satisfied then + for all columns above, use rec_prefix_field_no instead of + rec_field_no, and skip the clustered lookup below. */ + for (ulint i = 0; i < prebuilt->n_template; i++) { + mysql_row_templ_t* templ = prebuilt->mysql_template + i; + templ->rec_field_no = templ->rec_prefix_field_no; + ut_a(templ->rec_field_no != ULINT_UNDEFINED); + } + + srv_stats.n_sec_rec_cluster_reads_avoided.inc(); + return true; +} + /********************************************************************//** Searches for rows in the database. This is used in the interface to MySQL. This function opens a cursor, and also implements fetch next @@ -3729,7 +3843,6 @@ row_search_for_mysql( ulint* offsets = offsets_; ibool table_lock_waited = FALSE; byte* next_buf = 0; - ibool use_clustered_index = FALSE; rec_offs_init(offsets_); @@ -4790,69 +4903,10 @@ locks_ok: break; } - /* Get the clustered index record if needed, if we did not do the - search using the clustered index... */ - - use_clustered_index = - (index != clust_index && prebuilt->need_to_access_clustered); - - if (use_clustered_index && srv_prefix_index_cluster_optimization - && prebuilt->n_template <= index->n_fields) { - /* ...but, perhaps avoid the clustered index lookup if - all of the following are true: - 1) all columns are in the secondary index - 2) all values for columns that are prefix-only - indexes are shorter than the prefix size - This optimization can avoid many IOs for certain schemas. - */ - ibool row_contains_all_values = TRUE; - int i; - for (i = 0; i < prebuilt->n_template; i++) { - /* Condition (1) from above: is the field in the - index (prefix or not)? */ - mysql_row_templ_t* templ = - prebuilt->mysql_template + i; - ulint secondary_index_field_no = - templ->rec_prefix_field_no; - if (secondary_index_field_no == ULINT_UNDEFINED) { - row_contains_all_values = FALSE; - break; - } - /* Condition (2) from above: if this is a - prefix, is this row's value size shorter - than the prefix? */ - if (templ->rec_field_is_prefix) { - ulint record_size = rec_offs_nth_size( - offsets, - secondary_index_field_no); - const dict_field_t *field = - dict_index_get_nth_field( - index, - secondary_index_field_no); - ut_a(field->prefix_len > 0); - if (record_size >= field->prefix_len) { - row_contains_all_values = FALSE; - break; - } - } + if (index != clust_index && prebuilt->need_to_access_clustered) { + if (row_search_with_covering_prefix(prebuilt, rec, offsets)) { + goto use_covering_index; } - /* If (1) and (2) were true for all columns above, use - rec_prefix_field_no instead of rec_field_no, and skip - the clustered lookup below. */ - if (row_contains_all_values) { - for (i = 0; i < prebuilt->n_template; i++) { - mysql_row_templ_t* templ = - prebuilt->mysql_template + i; - templ->rec_field_no = - templ->rec_prefix_field_no; - ut_a(templ->rec_field_no != ULINT_UNDEFINED); - } - use_clustered_index = FALSE; - srv_stats.n_sec_rec_cluster_reads_avoided.inc(); - } - } - - if (use_clustered_index) { requires_clust_rec: ut_ad(index != clust_index); /* We use a 'goto' to the preceding label if a consistent @@ -4938,6 +4992,7 @@ requires_clust_rec: } } } else { +use_covering_index: result_rec = rec; } diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index b81ea60a413..97007c1107c 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -3685,6 +3685,117 @@ row_search_idx_cond_check( return(result); } +/** Return the record field length in characters. +@param[in] col table column of the field +@param[in] field_no field number +@param[in] rec physical record +@param[in] offsets field offsets in the physical record +@return field length in characters. */ +static +size_t +rec_field_len_in_chars( + const dict_col_t* col, + const ulint field_no, + const rec_t* rec, + const ulint* offsets) +{ + const ulint cset = dtype_get_charset_coll(col->prtype); + const CHARSET_INFO* cs = all_charsets[cset]; + ulint rec_field_len; + const char* rec_field = reinterpret_cast( + rec_get_nth_field( + rec, offsets, field_no, &rec_field_len)); + + if (UNIV_UNLIKELY(!cs)) { + ib_logf(IB_LOG_LEVEL_WARN, "Missing collation " ULINTPF, cset); + return SIZE_T_MAX; + } + + return(cs->cset->numchars(cs, rec_field, rec_field + rec_field_len)); +} + + +/** Avoid the clustered index lookup if all the following conditions +are true: +1) all columns are in secondary index +2) all values for columns that are prefix-only indexes are shorter +than the prefix size. This optimization can avoid many IOs for certain schemas. +@return true, to avoid clustered index lookup. */ +static +bool row_search_with_covering_prefix( + row_prebuilt_t* prebuilt, + const rec_t* rec, + const ulint* offsets) +{ + const dict_index_t* index = prebuilt->index; + ut_ad(!dict_index_is_clust(index)); + + if (!srv_prefix_index_cluster_optimization) { + return false; + } + + /** Optimization only applicable if the number of secondary index + fields are greater than or equal to number of clustered index fields. */ + if (prebuilt->n_template > index->n_fields) { + return false; + } + + for (ulint i = 0; i < prebuilt->n_template; i++) { + mysql_row_templ_t* templ = prebuilt->mysql_template + i; + ulint j = templ->rec_prefix_field_no; + + /** Condition (1) : is the field in the index. */ + if (j == ULINT_UNDEFINED) { + return false; + } + + /** Condition (2): If this is a prefix index then + row's value size shorter than prefix length. */ + + if (!templ->rec_field_is_prefix) { + continue; + } + + ulint rec_size = rec_offs_nth_size(offsets, j); + const dict_field_t* field = dict_index_get_nth_field(index, j); + ulint max_chars = field->prefix_len / templ->mbmaxlen; + + ut_a(field->prefix_len > 0); + + if (rec_size < max_chars) { + /* Record in bytes shorter than the index + prefix length in char. */ + continue; + } + + if (rec_size * templ->mbminlen >= field->prefix_len) { + /* Shortest representation string by the + byte length of the record is longer than the + maximum possible index prefix. */ + return false; + } + + + size_t num_chars = rec_field_len_in_chars( + field->col, j, rec, offsets); + + if (num_chars >= max_chars) { + /* No of chars to store the record exceeds + the index prefix character length. */ + return false; + } + } + + for (ulint i = 0; i < prebuilt->n_template; i++) { + mysql_row_templ_t* templ = prebuilt->mysql_template + i; + templ->rec_field_no = templ->rec_prefix_field_no; + ut_a(templ->rec_field_no != ULINT_UNDEFINED); + } + + srv_stats.n_sec_rec_cluster_reads_avoided.inc(); + return true; +} + /********************************************************************//** Searches for rows in the database. This is used in the interface to MySQL. This function opens a cursor, and also implements fetch next @@ -3748,7 +3859,6 @@ row_search_for_mysql( ulint* offsets = offsets_; ibool table_lock_waited = FALSE; byte* next_buf = 0; - bool use_clustered_index = false; rec_offs_init(offsets_); @@ -4810,71 +4920,10 @@ locks_ok: break; } - /* Get the clustered index record if needed, if we did not do the - search using the clustered index... */ - - use_clustered_index = - (index != clust_index && prebuilt->need_to_access_clustered); - - if (use_clustered_index && srv_prefix_index_cluster_optimization - && prebuilt->n_template <= index->n_fields) { - /* ...but, perhaps avoid the clustered index lookup if - all of the following are true: - 1) all columns are in the secondary index - 2) all values for columns that are prefix-only - indexes are shorter than the prefix size - This optimization can avoid many IOs for certain schemas. - */ - bool row_contains_all_values = true; - unsigned int i; - for (i = 0; i < prebuilt->n_template; i++) { - /* Condition (1) from above: is the field in the - index (prefix or not)? */ - const mysql_row_templ_t* templ = - prebuilt->mysql_template + i; - ulint secondary_index_field_no = - templ->rec_prefix_field_no; - if (secondary_index_field_no == ULINT_UNDEFINED) { - row_contains_all_values = false; - break; - } - /* Condition (2) from above: if this is a - prefix, is this row's value size shorter - than the prefix? */ - if (templ->rec_field_is_prefix) { - ulint record_size = rec_offs_nth_size( - offsets, - secondary_index_field_no); - const dict_field_t *field = - dict_index_get_nth_field( - index, - secondary_index_field_no); - ut_a(field->prefix_len > 0); - if (record_size >= field->prefix_len - / templ->mbmaxlen) { - row_contains_all_values = false; - break; - } - } + if (index != clust_index && prebuilt->need_to_access_clustered) { + if (row_search_with_covering_prefix(prebuilt, rec, offsets)) { + goto use_covering_index; } - /* If (1) and (2) were true for all columns above, use - rec_prefix_field_no instead of rec_field_no, and skip - the clustered lookup below. */ - if (row_contains_all_values) { - for (i = 0; i < prebuilt->n_template; i++) { - mysql_row_templ_t* templ = - prebuilt->mysql_template + i; - templ->rec_field_no = - templ->rec_prefix_field_no; - ut_a(templ->rec_field_no != ULINT_UNDEFINED); - } - use_clustered_index = false; - srv_stats.n_sec_rec_cluster_reads_avoided.inc(); - } - } - - if (use_clustered_index) { - requires_clust_rec: ut_ad(index != clust_index); /* We use a 'goto' to the preceding label if a consistent @@ -4960,6 +5009,7 @@ requires_clust_rec: } } } else { +use_covering_index: result_rec = rec; }