From aaaed9baa054e8bf8243fbb384c606306ea584b4 Mon Sep 17 00:00:00 2001 From: Alexey Bychko Date: Tue, 22 Jun 2021 18:57:01 +0700 Subject: [PATCH 01/15] MDEV-25960 yum update overwrites customized logrotation config (/etc/logrotate.d/mysql) added %config directive to /etc/logrotate.d/mysql and put it to server package. if file is edited by customer and defaults are changed - new .rpmnew file with defaults will be created next to old one. --- cmake/cpack_rpm.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 1eded9cb48e..d8cd97d5043 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -125,7 +125,11 @@ SET(ignored "%ignore ${CMAKE_INSTALL_PREFIX}/share/pkgconfig" ) -SET(CPACK_RPM_server_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSCONF2DIR}/*") +SET(CPACK_RPM_server_USER_FILELIST + ${ignored} + "%config(noreplace) ${INSTALL_SYSCONF2DIR}/*" + "%config(noreplace) ${INSTALL_SYSCONFDIR}/logrotate.d/mysql" + ) SET(CPACK_RPM_common_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSCONFDIR}/my.cnf") SET(CPACK_RPM_shared_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSCONF2DIR}/*") SET(CPACK_RPM_client_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSCONF2DIR}/*") From f67aee000df284027a6ecbd0fa639da56ecf8545 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 22 Jun 2021 22:00:23 -0400 Subject: [PATCH 02/15] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 07913307754..456c22d6a3f 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=2 -MYSQL_VERSION_PATCH=39 +MYSQL_VERSION_PATCH=40 From 83464029cec28b4463309e18fa85974798719932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 18 Jun 2021 10:16:38 +0300 Subject: [PATCH 03/15] Fix try for Galera test lp1376747-4 --- mysql-test/suite/galera/r/lp1376747-4.result | 2 +- mysql-test/suite/galera/t/lp1376747-4.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/r/lp1376747-4.result b/mysql-test/suite/galera/r/lp1376747-4.result index 3370e1d3d8e..efce740ea88 100644 --- a/mysql-test/suite/galera/r/lp1376747-4.result +++ b/mysql-test/suite/galera/r/lp1376747-4.result @@ -27,8 +27,8 @@ t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 -set debug_sync= 'RESET'; connection node_2a; +set debug_sync= 'RESET'; UNLOCK TABLES; SET SESSION wsrep_sync_wait = DEFAULT; SHOW CREATE TABLE t1; diff --git a/mysql-test/suite/galera/t/lp1376747-4.test b/mysql-test/suite/galera/t/lp1376747-4.test index d19ff422ab0..724901da068 100644 --- a/mysql-test/suite/galera/t/lp1376747-4.test +++ b/mysql-test/suite/galera/t/lp1376747-4.test @@ -46,10 +46,10 @@ SET debug_sync='now SIGNAL go2'; # the cluster as there is new FTRL that is still pausing it. UNLOCK TABLES; SHOW CREATE TABLE t1; -set debug_sync= 'RESET'; --connection node_2a --reap +set debug_sync= 'RESET'; UNLOCK TABLES; SET SESSION wsrep_sync_wait = DEFAULT; From 9258cfa4b469ab0d841e32ced7e501e04043637f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 22 Jun 2021 15:44:44 +0300 Subject: [PATCH 04/15] MDEV-25978 : rsync SST does not work with custom binlog name wsrep_sst_common did not correctly set name for binlog index file if custom binlog name was used and this name was not added to script command line. Added test case for both log_basename and log_binlog. --- .../r/galera_sst_rsync_binlogname.result | 96 +++++++++++++++++++ .../r/galera_sst_rsync_logbasename.result | 96 +++++++++++++++++++ .../galera/t/galera_sst_rsync_binlogname.cnf | 12 +++ .../galera/t/galera_sst_rsync_binlogname.test | 9 ++ .../galera/t/galera_sst_rsync_logbasename.cnf | 15 +++ .../t/galera_sst_rsync_logbasename.test | 9 ++ scripts/wsrep_sst_common.sh | 11 ++- sql/wsrep_sst.cc | 17 +++- 8 files changed, 259 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_sst_rsync_binlogname.result create mode 100644 mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_binlogname.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_binlogname.test create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_logbasename.test diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_binlogname.result b/mysql-test/suite/galera/r/galera_sst_rsync_binlogname.result new file mode 100644 index 00000000000..0f0a673947c --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_rsync_binlogname.result @@ -0,0 +1,96 @@ +connection node_1; +connection node_2; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result b/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result new file mode 100644 index 00000000000..0f0a673947c --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result @@ -0,0 +1,96 @@ +connection node_1; +connection node_2; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.cnf new file mode 100644 index 00000000000..b1e4278dceb --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.cnf @@ -0,0 +1,12 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=rsync + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' +log_bin=server1_binlog + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' +log_bin=server2_binlog diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.test b/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.test new file mode 100644 index 00000000000..b4ad6c43a0b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.test @@ -0,0 +1,9 @@ +--source include/galera_cluster.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--source suite/galera/include/galera_st_shutdown_slave.inc + +--source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf new file mode 100644 index 00000000000..4f25af7cd8b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf @@ -0,0 +1,15 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=rsync + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' +log_basename=server1 +log_bin + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' +log_basename=server2 +log_bin + diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.test b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.test new file mode 100644 index 00000000000..b4ad6c43a0b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.test @@ -0,0 +1,9 @@ +--source include/galera_cluster.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--source suite/galera/include/galera_st_shutdown_slave.inc + +--source include/auto_increment_offset_restore.inc diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 4dedecb439f..bb9a3b9f1b6 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -465,8 +465,9 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # Take the default name: - readonly WSREP_SST_OPT_BINLOG='mysql-bin' + # the default name, note that base of this name + # is already defined above + readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi fi @@ -549,9 +550,9 @@ get_binlog() # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # If the base name not specified, then we take - # the default name: - readonly WSREP_SST_OPT_BINLOG_INDEX='mysql-bin.index' + # the default name, note that base of this name + # is already defined above + readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi fi fi diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index e844086f5e3..2258a4d8f6d 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -990,6 +990,8 @@ static ssize_t sst_prepare_other (const char* method, { WSREP_ERROR("sst_prepare_other(): generate_binlog_index_opt_val() failed %d", ret); + if (binlog_opt_val) my_free(binlog_opt_val); + return ret; } make_wsrep_defaults_file(); @@ -1007,6 +1009,7 @@ static ssize_t sst_prepare_other (const char* method, wsrep_defaults_file, (int)getpid(), binlog_opt_val, binlog_index_opt_val); + my_free(binlog_opt_val); my_free(binlog_index_opt_val); @@ -1655,6 +1658,7 @@ static int sst_donate_other (const char* method, } char* binlog_opt_val= NULL; + char* binlog_index_opt_val= NULL; int ret; if ((ret= generate_binlog_opt_val(&binlog_opt_val))) @@ -1663,6 +1667,14 @@ static int sst_donate_other (const char* method, return ret; } + if ((ret= generate_binlog_index_opt_val(&binlog_index_opt_val))) + { + WSREP_ERROR("sst_prepare_other(): generate_binlog_index_opt_val() failed %d", + ret); + if (binlog_opt_val) my_free(binlog_opt_val); + return ret; + } + make_wsrep_defaults_file(); ret= snprintf (cmd_str(), cmd_len, @@ -1676,14 +1688,17 @@ static int sst_donate_other (const char* method, WSREP_SST_OPT_GTID " '%s:%lld' " WSREP_SST_OPT_GTID_DOMAIN_ID " '%d'" "%s" + "%s" "%s", method, addr, mysqld_port, mysqld_unix_port, mysql_real_data_home, wsrep_defaults_file, uuid, (long long) seqno, wsrep_gtid_domain_id, - binlog_opt_val, + binlog_opt_val, binlog_index_opt_val, bypass ? " " WSREP_SST_OPT_BYPASS : ""); + my_free(binlog_opt_val); + my_free(binlog_index_opt_val); if (ret < 0 || size_t(ret) >= cmd_len) { From 4ad148b148cfbb6f78b33ad9a7662f47c24cb759 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Fri, 25 Jun 2021 06:48:17 +0200 Subject: [PATCH 05/15] MDEV-26019: Upgrading MariaDB breaks TLS mariabackup SST Fixed typo in variable name that breaks SST in some scenatios. Also fixed one small inaccuracy after MDEV-25978 which leads to the use of an uninitialized variable when the --log-bin option is specified without an argument. --- scripts/wsrep_sst_common.sh | 11 +++++------ scripts/wsrep_sst_mariabackup.sh | 2 +- scripts/wsrep_sst_xtrabackup-v2.sh | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index bb9a3b9f1b6..562f9dc3aac 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -456,7 +456,7 @@ if [ -n "${MYSQLD_OPT_LOG_BASENAME:-}" -a \ fi # If the --log-bin option is present without a value, then -# setting WSREP_SST_OPT_BINLOG by using other arguments: +# set WSREP_SST_OPT_BINLOG value using other arguments: if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but @@ -465,9 +465,8 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # the default name, note that base of this name - # is already defined above - readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" + # Take the default name: + readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -550,8 +549,8 @@ get_binlog() # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # the default name, note that base of this name - # is already defined above + # the default name (note that base of this name + # is already defined above): readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi fi diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 339a8fcf4a5..46804c9dce4 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -386,7 +386,7 @@ get_transfer() elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then CN_option=',commonname=localhost' else - CN_option=",commonname='$WSREP_SST_OPT_HOST_UNSECAPED'" + CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'" fi tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'$CN_option$sockopt" wsrep_log_info "$action with cert=$tpem, key=$tkey, cafile=$tcert" diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index d76dc346a82..9600848dc77 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -388,7 +388,7 @@ get_transfer() elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then CN_option=',commonname=localhost' else - CN_option=",commonname='$WSREP_SST_OPT_HOST_UNSECAPED'" + CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'" fi tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'$CN_option$sockopt" wsrep_log_info "$action with cert=$tpem, key=$tkey, cafile=$tcert" From 05a4996c5c04d7ec4fe2ed2a5533685a688eccc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 22 Jun 2021 15:44:44 +0300 Subject: [PATCH 06/15] MDEV-25978 : rsync SST does not work with custom binlog name wsrep_sst_common did not correctly set name for binlog index file if custom binlog name was used and this name was not added to script command line. Added test case for both log_basename and log_binlog. --- .../r/galera_sst_rsync_binlogname.result | 96 +++++++++++++++++++ .../r/galera_sst_rsync_logbasename.result | 96 +++++++++++++++++++ .../galera/t/galera_sst_rsync_binlogname.cnf | 12 +++ .../galera/t/galera_sst_rsync_binlogname.test | 9 ++ .../galera/t/galera_sst_rsync_logbasename.cnf | 15 +++ .../t/galera_sst_rsync_logbasename.test | 9 ++ scripts/wsrep_sst_common.sh | 11 ++- sql/wsrep_sst.cc | 17 +++- 8 files changed, 259 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_sst_rsync_binlogname.result create mode 100644 mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_binlogname.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_binlogname.test create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_logbasename.test diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_binlogname.result b/mysql-test/suite/galera/r/galera_sst_rsync_binlogname.result new file mode 100644 index 00000000000..0f0a673947c --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_rsync_binlogname.result @@ -0,0 +1,96 @@ +connection node_1; +connection node_2; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result b/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result new file mode 100644 index 00000000000..0f0a673947c --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result @@ -0,0 +1,96 @@ +connection node_1; +connection node_2; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.cnf new file mode 100644 index 00000000000..b1e4278dceb --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.cnf @@ -0,0 +1,12 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=rsync + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' +log_bin=server1_binlog + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' +log_bin=server2_binlog diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.test b/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.test new file mode 100644 index 00000000000..b4ad6c43a0b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_binlogname.test @@ -0,0 +1,9 @@ +--source include/galera_cluster.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--source suite/galera/include/galera_st_shutdown_slave.inc + +--source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf new file mode 100644 index 00000000000..4f25af7cd8b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf @@ -0,0 +1,15 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=rsync + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' +log_basename=server1 +log_bin + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' +log_basename=server2 +log_bin + diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.test b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.test new file mode 100644 index 00000000000..b4ad6c43a0b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.test @@ -0,0 +1,9 @@ +--source include/galera_cluster.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--source suite/galera/include/galera_st_shutdown_slave.inc + +--source include/auto_increment_offset_restore.inc diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 4dedecb439f..bb9a3b9f1b6 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -465,8 +465,9 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # Take the default name: - readonly WSREP_SST_OPT_BINLOG='mysql-bin' + # the default name, note that base of this name + # is already defined above + readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi fi @@ -549,9 +550,9 @@ get_binlog() # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # If the base name not specified, then we take - # the default name: - readonly WSREP_SST_OPT_BINLOG_INDEX='mysql-bin.index' + # the default name, note that base of this name + # is already defined above + readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi fi fi diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index b18315dd160..dbebe91ec5b 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -984,6 +984,8 @@ static ssize_t sst_prepare_other (const char* method, { WSREP_ERROR("sst_prepare_other(): generate_binlog_index_opt_val() failed %d", ret); + if (binlog_opt_val) my_free(binlog_opt_val); + return ret; } make_wsrep_defaults_file(); @@ -1001,6 +1003,7 @@ static ssize_t sst_prepare_other (const char* method, wsrep_defaults_file, (int)getpid(), binlog_opt_val, binlog_index_opt_val); + my_free(binlog_opt_val); my_free(binlog_index_opt_val); @@ -1658,6 +1661,7 @@ static int sst_donate_other (const char* method, } char* binlog_opt_val= NULL; + char* binlog_index_opt_val= NULL; int ret; if ((ret= generate_binlog_opt_val(&binlog_opt_val))) @@ -1666,6 +1670,14 @@ static int sst_donate_other (const char* method, return ret; } + if ((ret= generate_binlog_index_opt_val(&binlog_index_opt_val))) + { + WSREP_ERROR("sst_prepare_other(): generate_binlog_index_opt_val() failed %d", + ret); + if (binlog_opt_val) my_free(binlog_opt_val); + return ret; + } + make_wsrep_defaults_file(); ret= snprintf (cmd_str(), cmd_len, @@ -1679,14 +1691,17 @@ static int sst_donate_other (const char* method, WSREP_SST_OPT_GTID " '%s:%lld' " WSREP_SST_OPT_GTID_DOMAIN_ID " '%d'" "%s" + "%s" "%s", method, addr, mysqld_port, mysqld_unix_port, mysql_real_data_home, wsrep_defaults_file, uuid, (long long) seqno, wsrep_gtid_domain_id, - binlog_opt_val, + binlog_opt_val, binlog_index_opt_val, bypass ? " " WSREP_SST_OPT_BYPASS : ""); + my_free(binlog_opt_val); + my_free(binlog_index_opt_val); if (ret < 0 || size_t(ret) >= cmd_len) { From 29098083f7ac3b445ee59c3e765eb634ec70b947 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Fri, 25 Jun 2021 06:48:17 +0200 Subject: [PATCH 07/15] MDEV-26019: Upgrading MariaDB breaks TLS mariabackup SST Fixed typo in variable name that breaks SST in some scenatios. Also fixed one small inaccuracy after MDEV-25978 which leads to the use of an uninitialized variable when the --log-bin option is specified without an argument. --- scripts/wsrep_sst_common.sh | 11 +++++------ scripts/wsrep_sst_mariabackup.sh | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index bb9a3b9f1b6..562f9dc3aac 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -456,7 +456,7 @@ if [ -n "${MYSQLD_OPT_LOG_BASENAME:-}" -a \ fi # If the --log-bin option is present without a value, then -# setting WSREP_SST_OPT_BINLOG by using other arguments: +# set WSREP_SST_OPT_BINLOG value using other arguments: if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but @@ -465,9 +465,8 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # the default name, note that base of this name - # is already defined above - readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" + # Take the default name: + readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -550,8 +549,8 @@ get_binlog() # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # the default name, note that base of this name - # is already defined above + # the default name (note that base of this name + # is already defined above): readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi fi diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 339a8fcf4a5..46804c9dce4 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -386,7 +386,7 @@ get_transfer() elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then CN_option=',commonname=localhost' else - CN_option=",commonname='$WSREP_SST_OPT_HOST_UNSECAPED'" + CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'" fi tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'$CN_option$sockopt" wsrep_log_info "$action with cert=$tpem, key=$tkey, cafile=$tcert" From 12c80df4825955253a1a442098dbceb2a3e59971 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 25 Jun 2021 18:03:29 -0700 Subject: [PATCH 08/15] MDEV-20411 Procedure containing CTE incorrectly stored in mysql.proc If the first token of the body of a stored procedure was 'WITH' then the beginning of the body was determined incorrectly and that token was missing in the string representing the body of the SP in mysql.proc. As a resultnany call of such procedure failed as the string representing the body could not be parsed. The patch corrects the code of the functions get_tok_start() and get_cpp_tok_start() of the class Lex_input_stream to make them take into account look ahead tokens. The patch is needed only for 10.2 as this problem has neen resolved in 10.3+. --- mysql-test/r/cte_nonrecursive.result | 55 ++++++++++++++++++++++++++++ mysql-test/r/fulltext.result | 2 +- mysql-test/t/cte_nonrecursive.test | 29 +++++++++++++++ sql/sql_lex.h | 4 +- 4 files changed, 87 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index 7c6c6e8dedd..c1d7fd0a615 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1964,4 +1964,59 @@ call p1(); ERROR 42S22: Unknown column 'a' in 'field list' drop procedure p1; drop table t1,t2; +# +# MDEV-20411: SP containing only one SELECT with WITH clause +# +create procedure sp1 () +with cte as (select 1 as a) select * from cte; +call sp1(); +a +1 +call sp1(); +a +1 +create table t1 (a int); +insert into t1 values (3), (7), (1), (7), (1), (1), (3), (1), (5); +create procedure sp2 () +with cte as (select * from t1) select * from cte; +call sp2(); +a +3 +7 +1 +7 +1 +1 +3 +1 +5 +call sp2(); +a +3 +7 +1 +7 +1 +1 +3 +1 +5 +create procedure sp3 () +with cte as (select * from t1 group by a) select * from cte; +call sp3(); +a +1 +3 +5 +7 +call sp3(); +a +1 +3 +5 +7 +drop procedure sp1; +drop procedure sp2; +drop procedure sp3; +drop table t1; # End of 10.2 tests diff --git a/mysql-test/r/fulltext.result b/mysql-test/r/fulltext.result index 5118cc38fdf..709e91cca4d 100644 --- a/mysql-test/r/fulltext.result +++ b/mysql-test/r/fulltext.result @@ -49,7 +49,7 @@ a b Full-text indexes are called collections Only MyISAM tables support collections select * from t1 where MATCH(a,b) AGAINST ("indexes" IN BOOLEAN MODE WITH QUERY EXPANSION); -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'QUERY EXPANSION)' at line 1 +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH QUERY EXPANSION)' at line 1 explain select * from t1 where MATCH(a,b) AGAINST ("collections"); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 fulltext a a 0 1 Using where diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index f994781548f..cbe4f8b855d 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -1463,4 +1463,33 @@ drop procedure p1; drop table t1,t2; + +--echo # +--echo # MDEV-20411: SP containing only one SELECT with WITH clause +--echo # + +create procedure sp1 () +with cte as (select 1 as a) select * from cte; +call sp1(); +call sp1(); + +create table t1 (a int); +insert into t1 values (3), (7), (1), (7), (1), (1), (3), (1), (5); + +create procedure sp2 () +with cte as (select * from t1) select * from cte; +call sp2(); +call sp2(); + +create procedure sp3 () +with cte as (select * from t1 group by a) select * from cte; +call sp3(); +call sp3(); + +drop procedure sp1; +drop procedure sp2; +drop procedure sp3; + +drop table t1; + --echo # End of 10.2 tests diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 03c06b9ae0f..bdf52e8ef7b 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -2176,7 +2176,7 @@ public: /** Get the token start position, in the raw buffer. */ const char *get_tok_start() { - return m_tok_start; + return lookahead_token >= 0 ? m_tok_start_prev : m_tok_start; } void set_cpp_tok_start(const char *pos) @@ -2222,7 +2222,7 @@ public: /** Get the token start position, in the pre-processed buffer. */ const char *get_cpp_tok_start() { - return m_cpp_tok_start; + return lookahead_token >= 0 ? m_cpp_tok_start_prev : m_cpp_tok_start; } /** Get the token end position, in the pre-processed buffer. */ From 8b3f816cab30eadd574b975d11c9e525a0b1691e Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sat, 26 Jun 2021 08:51:17 -0700 Subject: [PATCH 09/15] Adjusted test results after the fix for MDEV-20411 --- mysql-test/suite/funcs_1/r/innodb_views.result | 8 ++++---- mysql-test/suite/funcs_1/r/memory_views.result | 8 ++++---- mysql-test/suite/funcs_1/r/storedproc.result | 11 +++++------ mysql-test/suite/innodb_fts/r/fulltext.result | 2 +- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/mysql-test/suite/funcs_1/r/innodb_views.result b/mysql-test/suite/funcs_1/r/innodb_views.result index 1316988ffc3..5e4b008326d 100644 --- a/mysql-test/suite/funcs_1/r/innodb_views.result +++ b/mysql-test/suite/funcs_1/r/innodb_views.result @@ -3551,11 +3551,11 @@ CREATE VIEW v1 or REPLACE AS Select * from tb2 my_table; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'or REPLACE AS Select * from tb2 my_table' at line 1 CREATE VIEW v1 WITH CASCADED CHECK OPTION AS Select * from tb2 my_table limit 50; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION AS Select * +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS Select * from tb2 my_table limit 50' at line 1 CREATE VIEW v1 WITH LOCAL CHECK OPTION AS Select * from tb2 my_table limit 50; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'LOCAL CHECK OPTION AS Select * +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH LOCAL CHECK OPTION AS Select * from tb2 my_table limit 50' at line 1 SELECT * FROM tb2 my_table CREATE VIEW As v1; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CREATE VIEW As v1' at line 1 @@ -3585,7 +3585,7 @@ FROM test.tb2 my_table CHECK OPTION WITH CASCADED; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CHECK OPTION WITH CASCADED' at line 2 CREATE OR REPLACE VIEW v1 WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table' at line 1 CREATE OR REPLACE AS SELECT F59, F60 FROM test.tb2 my_table VIEW v1 WITH CASCADED CHECK OPTION; @@ -3614,7 +3614,7 @@ FROM test.tb2 my_table CHECK OPTION WITH LOCAL; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CHECK OPTION WITH LOCAL' at line 2 CREATE OR REPLACE VIEW v1 WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table' at line 1 CREATE OR REPLACE AS SELECT F59, F60 FROM test.tb2 my_table VIEW v1 WITH LOCAL CHECK OPTION; diff --git a/mysql-test/suite/funcs_1/r/memory_views.result b/mysql-test/suite/funcs_1/r/memory_views.result index 9f3fbd7bb1a..acd042511a0 100644 --- a/mysql-test/suite/funcs_1/r/memory_views.result +++ b/mysql-test/suite/funcs_1/r/memory_views.result @@ -3552,11 +3552,11 @@ CREATE VIEW v1 or REPLACE AS Select * from tb2 my_table; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'or REPLACE AS Select * from tb2 my_table' at line 1 CREATE VIEW v1 WITH CASCADED CHECK OPTION AS Select * from tb2 my_table limit 50; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION AS Select * +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS Select * from tb2 my_table limit 50' at line 1 CREATE VIEW v1 WITH LOCAL CHECK OPTION AS Select * from tb2 my_table limit 50; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'LOCAL CHECK OPTION AS Select * +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH LOCAL CHECK OPTION AS Select * from tb2 my_table limit 50' at line 1 SELECT * FROM tb2 my_table CREATE VIEW As v1; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CREATE VIEW As v1' at line 1 @@ -3586,7 +3586,7 @@ FROM test.tb2 my_table CHECK OPTION WITH CASCADED; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CHECK OPTION WITH CASCADED' at line 2 CREATE OR REPLACE VIEW v1 WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table' at line 1 CREATE OR REPLACE AS SELECT F59, F60 FROM test.tb2 my_table VIEW v1 WITH CASCADED CHECK OPTION; @@ -3615,7 +3615,7 @@ FROM test.tb2 my_table CHECK OPTION WITH LOCAL; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CHECK OPTION WITH LOCAL' at line 2 CREATE OR REPLACE VIEW v1 WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table' at line 1 CREATE OR REPLACE AS SELECT F59, F60 FROM test.tb2 my_table VIEW v1 WITH LOCAL CHECK OPTION; diff --git a/mysql-test/suite/funcs_1/r/storedproc.result b/mysql-test/suite/funcs_1/r/storedproc.result index 516ea983563..64573ed2bd1 100644 --- a/mysql-test/suite/funcs_1/r/storedproc.result +++ b/mysql-test/suite/funcs_1/r/storedproc.result @@ -2807,7 +2807,7 @@ ERROR 42000: You have an error in your SQL syntax; check the manual that corresp SELECT * from t1 where f2=f1' at line 1 CREATE PROCEDURE with() SELECT * from t1 where f2=f1; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '() +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'with() SELECT * from t1 where f2=f1' at line 1 CREATE PROCEDURE write() SELECT * from t1 where f2=f1; @@ -9219,7 +9219,7 @@ CREATE PROCEDURE sp1() BEGIN declare with char; END// -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'char; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'with char; END' at line 3 DROP PROCEDURE IF EXISTS sp1; Warnings: @@ -11547,9 +11547,8 @@ BEGIN declare with condition for sqlstate '02000'; declare exit handler for with set @var2 = 1; END// -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'condition for sqlstate '02000'; -declare exit handler for with set @var2 = 1; -END' at line 3 +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'with condition for sqlstate '02000'; +declare exit handler for with set @var2 ...' at line 3 DROP PROCEDURE IF EXISTS sp1; Warnings: Note 1305 PROCEDURE db_storedproc.sp1 does not exist @@ -13649,7 +13648,7 @@ CREATE PROCEDURE sp1( ) BEGIN declare with handler for sqlstate '02000' set @var2 = 1; END// -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'handler for sqlstate '02000' set @var2 = 1; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'with handler for sqlstate '02000' set @var2 = 1; END' at line 3 DROP PROCEDURE IF EXISTS sp1; Warnings: diff --git a/mysql-test/suite/innodb_fts/r/fulltext.result b/mysql-test/suite/innodb_fts/r/fulltext.result index 90deb48a223..1e5a83fd44c 100644 --- a/mysql-test/suite/innodb_fts/r/fulltext.result +++ b/mysql-test/suite/innodb_fts/r/fulltext.result @@ -56,7 +56,7 @@ Only MyISAM tables support collections MySQL has now support for full-text search Full-text search in MySQL implements vector space model select * from t1 where MATCH(a,b) AGAINST ("indexes" IN BOOLEAN MODE WITH QUERY EXPANSION); -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'QUERY EXPANSION)' at line 1 +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH QUERY EXPANSION)' at line 1 explain select * from t1 where MATCH(a,b) AGAINST ("collections"); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 fulltext a a 0 1 Using where From 4e4f742ed7987ee52a34618d2ea3731e5e198ed8 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sat, 26 Jun 2021 23:11:10 -0700 Subject: [PATCH 10/15] Adjusted test results after the fix for MDEV-20411 (2) --- mysql-test/suite/funcs_1/r/myisam_views-big.result | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/funcs_1/r/myisam_views-big.result b/mysql-test/suite/funcs_1/r/myisam_views-big.result index a1d59e1c2f9..d3227d83b01 100644 --- a/mysql-test/suite/funcs_1/r/myisam_views-big.result +++ b/mysql-test/suite/funcs_1/r/myisam_views-big.result @@ -4054,11 +4054,11 @@ CREATE VIEW v1 or REPLACE AS Select * from tb2 my_table; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'or REPLACE AS Select * from tb2 my_table' at line 1 CREATE VIEW v1 WITH CASCADED CHECK OPTION AS Select * from tb2 my_table limit 50; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION AS Select * +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS Select * from tb2 my_table limit 50' at line 1 CREATE VIEW v1 WITH LOCAL CHECK OPTION AS Select * from tb2 my_table limit 50; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'LOCAL CHECK OPTION AS Select * +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH LOCAL CHECK OPTION AS Select * from tb2 my_table limit 50' at line 1 SELECT * FROM tb2 my_table CREATE VIEW As v1; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CREATE VIEW As v1' at line 1 @@ -4088,7 +4088,7 @@ FROM test.tb2 my_table CHECK OPTION WITH CASCADED; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CHECK OPTION WITH CASCADED' at line 2 CREATE OR REPLACE VIEW v1 WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table' at line 1 CREATE OR REPLACE AS SELECT F59, F60 FROM test.tb2 my_table VIEW v1 WITH CASCADED CHECK OPTION; @@ -4117,7 +4117,7 @@ FROM test.tb2 my_table CHECK OPTION WITH LOCAL; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CHECK OPTION WITH LOCAL' at line 2 CREATE OR REPLACE VIEW v1 WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table; -ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CASCADED CHECK OPTION +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH CASCADED CHECK OPTION AS SELECT F59, F60 FROM test.tb2 my_table' at line 1 CREATE OR REPLACE AS SELECT F59, F60 FROM test.tb2 my_table VIEW v1 WITH LOCAL CHECK OPTION; From 8147d2e6183a1a4a4f3db2884966f5da2d17678c Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 28 Jun 2021 11:52:00 +0400 Subject: [PATCH 11/15] MDEV-25461 Assertion `je->state == JST_KEY' failed in Geometry::create_from_json. Handle invalid GEOJSON-s in Geometry::create_from_json(). --- mysql-test/r/gis-json.result | 10 ++++++++++ mysql-test/t/gis-json.test | 7 +++++++ sql/spatial.cc | 6 ++++++ 3 files changed, 23 insertions(+) diff --git a/mysql-test/r/gis-json.result b/mysql-test/r/gis-json.result index e52a7c809c6..ace9e9e9ae2 100644 --- a/mysql-test/r/gis-json.result +++ b/mysql-test/r/gis-json.result @@ -107,6 +107,16 @@ Warning 4076 Incorrect GeoJSON format - empty 'coordinates' array. SELECT ST_GEOMFROMGEOJSON("{ \"type\": \"Feature\", \"geometry\": [10, 20] }"); ST_GEOMFROMGEOJSON("{ \"type\": \"Feature\", \"geometry\": [10, 20] }") NULL +SELECT ST_ASTEXT (ST_GEOMFROMGEOJSON ('{ "type": "GEOMETRYCOLLECTION", "coordinates": [102.0, 0.0]}')); +ST_ASTEXT (ST_GEOMFROMGEOJSON ('{ "type": "GEOMETRYCOLLECTION", "coordinates": [102.0, 0.0]}')) +NULL +Warnings: +Warning 4048 Incorrect GeoJSON format specified for st_geomfromgeojson function. +SELECT ST_ASTEXT(ST_GEOMFROMGEOJSON('{"type": ["POINT"], "coINates": [0,0] }')); +ST_ASTEXT(ST_GEOMFROMGEOJSON('{"type": ["POINT"], "coINates": [0,0] }')) +NULL +Warnings: +Warning 4048 Incorrect GeoJSON format specified for st_geomfromgeojson function. # # End of 10.2 tests # diff --git a/mysql-test/t/gis-json.test b/mysql-test/t/gis-json.test index a97e9411e5c..ff6298c50a6 100644 --- a/mysql-test/t/gis-json.test +++ b/mysql-test/t/gis-json.test @@ -46,6 +46,13 @@ SELECT st_astext(st_geomfromgeojson('{"type": "MultiPolygon","coordinates": []}' SELECT ST_GEOMFROMGEOJSON("{ \"type\": \"Feature\", \"geometry\": [10, 20] }"); +# +# MDEV-25461 Assertion `je->state == JST_KEY' failed in Geometry::create_from_json. +# + +SELECT ST_ASTEXT (ST_GEOMFROMGEOJSON ('{ "type": "GEOMETRYCOLLECTION", "coordinates": [102.0, 0.0]}')); + +SELECT ST_ASTEXT(ST_GEOMFROMGEOJSON('{"type": ["POINT"], "coINates": [0,0] }')); --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/spatial.cc b/sql/spatial.cc index 7bb3ecff948..8bd9acef18f 100644 --- a/sql/spatial.cc +++ b/sql/spatial.cc @@ -539,7 +539,11 @@ Geometry *Geometry::create_from_json(Geometry_buffer *buffer, goto handle_geometry_key; feature_type_found= 1; } + else /* can't understand the type. */ + break; } + else /* The "type" value can only be string. */ + break; } else if (key_len == coord_keyname_len && memcmp(key_buf, coord_keyname, coord_keyname_len) == 0) @@ -556,6 +560,8 @@ Geometry *Geometry::create_from_json(Geometry_buffer *buffer, coord_start= je->value_begin; if (ci && ci != &geometrycollection_class) goto create_geom; + if (json_skip_level(je)) + goto err_return; } } else if (key_len == geometries_keyname_len && From 58252fff15acfe7c7b0452a87e202e3f8e454e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 29 Jun 2021 14:28:23 +0300 Subject: [PATCH 12/15] MDEV-26040 os_file_set_size() may not work on O_DIRECT files os_file_set_size(): Trim the current size down to the file system block size, to obey the constraints for unbuffered I/O. --- storage/innobase/os/os0file.cc | 38 +++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index d2d5769d85e..884cadaf9f6 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -5420,6 +5420,8 @@ os_file_set_size( fallback: #else + struct stat statbuf; + if (is_sparse) { bool success = !ftruncate(file, size); if (!success) { @@ -5433,10 +5435,17 @@ fallback: # ifdef HAVE_POSIX_FALLOCATE int err; do { - os_offset_t current_size = os_file_get_size(file); - err = current_size >= size - ? 0 : posix_fallocate(file, current_size, + if (fstat(file, &statbuf)) { + err = errno; + } else { + os_offset_t current_size = statbuf.st_size; + if (current_size >= size) { + return true; + } + current_size &= ~os_offset_t(statbuf.st_blksize - 1); + err = posix_fallocate(file, current_size, size - current_size); + } } while (err == EINTR && srv_shutdown_state <= SRV_SHUTDOWN_INITIATED); @@ -5459,6 +5468,27 @@ fallback: # endif /* HAVE_POSIX_ALLOCATE */ #endif /* _WIN32*/ +#ifdef _WIN32 + os_offset_t current_size = os_file_get_size(file); + FILE_STORAGE_INFO info; + if (GetFileInformationByHandleEx(file, FileStorageInfo, &info, + sizeof info)) { + if (info.LogicalBytesPerSector) { + current_size &= ~os_offset_t(info.LogicalBytesPerSector + - 1); + } + } +#else + if (fstat(file, &statbuf)) { + return false; + } + os_offset_t current_size = statbuf.st_size + & ~os_offset_t(statbuf.st_blksize - 1); +#endif + if (current_size >= size) { + return true; + } + /* Write up to 1 megabyte at a time. */ ulint buf_size = ut_min( static_cast(64), @@ -5476,8 +5506,6 @@ fallback: /* Write buffer full of zeros */ memset(buf, 0, buf_size); - os_offset_t current_size = os_file_get_size(file); - while (current_size < size && srv_shutdown_state <= SRV_SHUTDOWN_INITIATED) { ulint n_bytes; From 768c51880a5aa6d25d4c0fe7de7a88561ff46422 Mon Sep 17 00:00:00 2001 From: "xing-zhi, jiang" Date: Thu, 24 Jun 2021 14:16:11 +0300 Subject: [PATCH 13/15] MDEV-25129 Add KEYWORDS view to the INFORMATION_SCHEMA Add KEYWORDS table and SQL_FUNCTIONS table to INFORMATION_SCHEMA. This commits needs some minor changes when propagated upwards (e.g. func_array in item_create.cc has a termination element that doesn't exist in later versions of MariaDB) --- .gitignore | 2 + mysql-test/r/information_schema.result | 2 + .../r/information_schema_all_engines.result | 12 ++- .../suite/funcs_1/r/is_columns_is.result | 4 + .../funcs_1/r/is_columns_is_embedded.result | 4 + .../suite/funcs_1/r/is_tables_is.result | 92 +++++++++++++++++++ .../funcs_1/r/is_tables_is_embedded.result | 92 +++++++++++++++++++ sql/handler.h | 2 + sql/item_create.cc | 11 +-- sql/item_create.h | 6 ++ sql/lex.h | 7 +- sql/sql_show.cc | 83 +++++++++++++++++ 12 files changed, 306 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 00754884d3d..dc8f6940fdf 100644 --- a/.gitignore +++ b/.gitignore @@ -517,3 +517,5 @@ compile_commands.json # Visual Studio Code workspace .vscode/ + +.cache/clangd diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index 1594ebf75e0..c7153bd6383 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -65,6 +65,7 @@ GEOMETRY_COLUMNS GLOBAL_STATUS GLOBAL_VARIABLES INDEX_STATISTICS +KEYWORDS KEY_CACHES KEY_COLUMN_USAGE PARAMETERS @@ -79,6 +80,7 @@ SCHEMA_PRIVILEGES SESSION_STATUS SESSION_VARIABLES SPATIAL_REF_SYS +SQL_FUNCTIONS STATISTICS SYSTEM_VARIABLES TABLES diff --git a/mysql-test/r/information_schema_all_engines.result b/mysql-test/r/information_schema_all_engines.result index 0ce9f15f753..542299ca5c1 100644 --- a/mysql-test/r/information_schema_all_engines.result +++ b/mysql-test/r/information_schema_all_engines.result @@ -41,6 +41,7 @@ INNODB_SYS_VIRTUAL INNODB_TABLESPACES_ENCRYPTION INNODB_TABLESPACES_SCRUBBING INNODB_TRX +KEYWORDS KEY_CACHES KEY_COLUMN_USAGE PARAMETERS @@ -55,6 +56,7 @@ SCHEMA_PRIVILEGES SESSION_STATUS SESSION_VARIABLES SPATIAL_REF_SYS +SQL_FUNCTIONS STATISTICS SYSTEM_VARIABLES TABLES @@ -121,6 +123,7 @@ INNODB_SYS_VIRTUAL TABLE_ID INNODB_TABLESPACES_ENCRYPTION SPACE INNODB_TABLESPACES_SCRUBBING SPACE INNODB_TRX trx_id +KEYWORDS WORD KEY_CACHES KEY_CACHE_NAME KEY_COLUMN_USAGE CONSTRAINT_SCHEMA PARAMETERS SPECIFIC_SCHEMA @@ -135,6 +138,7 @@ SCHEMA_PRIVILEGES TABLE_SCHEMA SESSION_STATUS VARIABLE_NAME SESSION_VARIABLES VARIABLE_NAME SPATIAL_REF_SYS SRID +SQL_FUNCTIONS FUNCTION STATISTICS TABLE_SCHEMA SYSTEM_VARIABLES VARIABLE_NAME TABLES TABLE_SCHEMA @@ -201,6 +205,7 @@ INNODB_SYS_VIRTUAL TABLE_ID INNODB_TABLESPACES_ENCRYPTION SPACE INNODB_TABLESPACES_SCRUBBING SPACE INNODB_TRX trx_id +KEYWORDS WORD KEY_CACHES KEY_CACHE_NAME KEY_COLUMN_USAGE CONSTRAINT_SCHEMA PARAMETERS SPECIFIC_SCHEMA @@ -215,6 +220,7 @@ SCHEMA_PRIVILEGES TABLE_SCHEMA SESSION_STATUS VARIABLE_NAME SESSION_VARIABLES VARIABLE_NAME SPATIAL_REF_SYS SRID +SQL_FUNCTIONS FUNCTION STATISTICS TABLE_SCHEMA SYSTEM_VARIABLES VARIABLE_NAME TABLES TABLE_SCHEMA @@ -356,6 +362,7 @@ Database: information_schema | INNODB_TABLESPACES_ENCRYPTION | | INNODB_TABLESPACES_SCRUBBING | | INNODB_TRX | +| KEYWORDS | | KEY_CACHES | | KEY_COLUMN_USAGE | | PARAMETERS | @@ -370,6 +377,7 @@ Database: information_schema | SESSION_STATUS | | SESSION_VARIABLES | | SPATIAL_REF_SYS | +| SQL_FUNCTIONS | | STATISTICS | | SYSTEM_VARIABLES | | TABLES | @@ -426,6 +434,7 @@ Database: INFORMATION_SCHEMA | INNODB_TABLESPACES_ENCRYPTION | | INNODB_TABLESPACES_SCRUBBING | | INNODB_TRX | +| KEYWORDS | | KEY_CACHES | | KEY_COLUMN_USAGE | | PARAMETERS | @@ -440,6 +449,7 @@ Database: INFORMATION_SCHEMA | SESSION_STATUS | | SESSION_VARIABLES | | SPATIAL_REF_SYS | +| SQL_FUNCTIONS | | STATISTICS | | SYSTEM_VARIABLES | | TABLES | @@ -459,5 +469,5 @@ Wildcard: inf_rmation_schema | information_schema | SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') GROUP BY TABLE_SCHEMA; table_schema count(*) -information_schema 65 +information_schema 67 mysql 30 diff --git a/mysql-test/suite/funcs_1/r/is_columns_is.result b/mysql-test/suite/funcs_1/r/is_columns_is.result index 672c8e0810a..3282d30669e 100644 --- a/mysql-test/suite/funcs_1/r/is_columns_is.result +++ b/mysql-test/suite/funcs_1/r/is_columns_is.result @@ -181,6 +181,7 @@ def information_schema INDEX_STATISTICS INDEX_NAME 3 '' NO varchar 192 576 NULL def information_schema INDEX_STATISTICS ROWS_READ 4 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(21) select NEVER NULL def information_schema INDEX_STATISTICS TABLE_NAME 2 '' NO varchar 192 576 NULL NULL NULL utf8 utf8_general_ci varchar(192) select NEVER NULL def information_schema INDEX_STATISTICS TABLE_SCHEMA 1 '' NO varchar 192 576 NULL NULL NULL utf8 utf8_general_ci varchar(192) select NEVER NULL +def information_schema KEYWORDS WORD 1 NULL YES varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select NEVER NULL def information_schema KEY_CACHES BLOCK_SIZE 5 0 NO bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned select NEVER NULL def information_schema KEY_CACHES DIRTY_BLOCKS 8 0 NO bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned select NEVER NULL def information_schema KEY_CACHES FULL_SIZE 4 0 NO bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned select NEVER NULL @@ -336,6 +337,7 @@ def information_schema SPATIAL_REF_SYS AUTH_NAME 2 '' NO varchar 512 1536 NULL N def information_schema SPATIAL_REF_SYS AUTH_SRID 3 0 NO int NULL NULL 10 0 NULL NULL NULL int(5) select NEVER NULL def information_schema SPATIAL_REF_SYS SRID 1 0 NO smallint NULL NULL 5 0 NULL NULL NULL smallint(5) select NEVER NULL def information_schema SPATIAL_REF_SYS SRTEXT 4 '' NO varchar 2048 6144 NULL NULL NULL utf8 utf8_general_ci varchar(2048) select NEVER NULL +def information_schema SQL_FUNCTIONS FUNCTION 1 NULL YES varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select NEVER NULL def information_schema STATISTICS CARDINALITY 10 NULL YES bigint NULL NULL 19 0 NULL NULL NULL bigint(21) select NEVER NULL def information_schema STATISTICS COLLATION 9 NULL YES varchar 1 3 NULL NULL NULL utf8 utf8_general_ci varchar(1) select NEVER NULL def information_schema STATISTICS COLUMN_NAME 8 '' NO varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select NEVER NULL @@ -716,6 +718,7 @@ NULL information_schema GEOMETRY_COLUMNS SRID smallint NULL NULL NULL NULL small 3.0000 information_schema INDEX_STATISTICS TABLE_NAME varchar 192 576 utf8 utf8_general_ci varchar(192) 3.0000 information_schema INDEX_STATISTICS INDEX_NAME varchar 192 576 utf8 utf8_general_ci varchar(192) NULL information_schema INDEX_STATISTICS ROWS_READ bigint NULL NULL NULL NULL bigint(21) +3.0000 information_schema KEYWORDS WORD varchar 64 192 utf8 utf8_general_ci varchar(64) 3.0000 information_schema KEY_CACHES KEY_CACHE_NAME varchar 192 576 utf8 utf8_general_ci varchar(192) NULL information_schema KEY_CACHES SEGMENTS int NULL NULL NULL NULL int(3) unsigned NULL information_schema KEY_CACHES SEGMENT_NUMBER int NULL NULL NULL NULL int(3) unsigned @@ -871,6 +874,7 @@ NULL information_schema SPATIAL_REF_SYS SRID smallint NULL NULL NULL NULL smalli 3.0000 information_schema SPATIAL_REF_SYS AUTH_NAME varchar 512 1536 utf8 utf8_general_ci varchar(512) NULL information_schema SPATIAL_REF_SYS AUTH_SRID int NULL NULL NULL NULL int(5) 3.0000 information_schema SPATIAL_REF_SYS SRTEXT varchar 2048 6144 utf8 utf8_general_ci varchar(2048) +3.0000 information_schema SQL_FUNCTIONS FUNCTION varchar 64 192 utf8 utf8_general_ci varchar(64) 3.0000 information_schema STATISTICS TABLE_CATALOG varchar 512 1536 utf8 utf8_general_ci varchar(512) 3.0000 information_schema STATISTICS TABLE_SCHEMA varchar 64 192 utf8 utf8_general_ci varchar(64) 3.0000 information_schema STATISTICS TABLE_NAME varchar 64 192 utf8 utf8_general_ci varchar(64) diff --git a/mysql-test/suite/funcs_1/r/is_columns_is_embedded.result b/mysql-test/suite/funcs_1/r/is_columns_is_embedded.result index 85026160b41..9116830e88c 100644 --- a/mysql-test/suite/funcs_1/r/is_columns_is_embedded.result +++ b/mysql-test/suite/funcs_1/r/is_columns_is_embedded.result @@ -181,6 +181,7 @@ def information_schema INDEX_STATISTICS INDEX_NAME 3 '' NO varchar 192 576 NULL def information_schema INDEX_STATISTICS ROWS_READ 4 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(21) NEVER NULL def information_schema INDEX_STATISTICS TABLE_NAME 2 '' NO varchar 192 576 NULL NULL NULL utf8 utf8_general_ci varchar(192) NEVER NULL def information_schema INDEX_STATISTICS TABLE_SCHEMA 1 '' NO varchar 192 576 NULL NULL NULL utf8 utf8_general_ci varchar(192) NEVER NULL +def information_schema KEYWORDS WORD 1 NULL YES varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) NEVER NULL def information_schema KEY_CACHES BLOCK_SIZE 5 0 NO bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned NEVER NULL def information_schema KEY_CACHES DIRTY_BLOCKS 8 0 NO bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned NEVER NULL def information_schema KEY_CACHES FULL_SIZE 4 0 NO bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned NEVER NULL @@ -336,6 +337,7 @@ def information_schema SPATIAL_REF_SYS AUTH_NAME 2 '' NO varchar 512 1536 NULL N def information_schema SPATIAL_REF_SYS AUTH_SRID 3 0 NO int NULL NULL 10 0 NULL NULL NULL int(5) NEVER NULL def information_schema SPATIAL_REF_SYS SRID 1 0 NO smallint NULL NULL 5 0 NULL NULL NULL smallint(5) NEVER NULL def information_schema SPATIAL_REF_SYS SRTEXT 4 '' NO varchar 2048 6144 NULL NULL NULL utf8 utf8_general_ci varchar(2048) NEVER NULL +def information_schema SQL_FUNCTIONS FUNCTION 1 NULL YES varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) NEVER NULL def information_schema STATISTICS CARDINALITY 10 NULL YES bigint NULL NULL 19 0 NULL NULL NULL bigint(21) NEVER NULL def information_schema STATISTICS COLLATION 9 NULL YES varchar 1 3 NULL NULL NULL utf8 utf8_general_ci varchar(1) NEVER NULL def information_schema STATISTICS COLUMN_NAME 8 '' NO varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) NEVER NULL @@ -716,6 +718,7 @@ NULL information_schema GEOMETRY_COLUMNS SRID smallint NULL NULL NULL NULL small 3.0000 information_schema INDEX_STATISTICS TABLE_NAME varchar 192 576 utf8 utf8_general_ci varchar(192) 3.0000 information_schema INDEX_STATISTICS INDEX_NAME varchar 192 576 utf8 utf8_general_ci varchar(192) NULL information_schema INDEX_STATISTICS ROWS_READ bigint NULL NULL NULL NULL bigint(21) +3.0000 information_schema KEYWORDS WORD varchar 64 192 utf8 utf8_general_ci varchar(64) 3.0000 information_schema KEY_CACHES KEY_CACHE_NAME varchar 192 576 utf8 utf8_general_ci varchar(192) NULL information_schema KEY_CACHES SEGMENTS int NULL NULL NULL NULL int(3) unsigned NULL information_schema KEY_CACHES SEGMENT_NUMBER int NULL NULL NULL NULL int(3) unsigned @@ -871,6 +874,7 @@ NULL information_schema SPATIAL_REF_SYS SRID smallint NULL NULL NULL NULL smalli 3.0000 information_schema SPATIAL_REF_SYS AUTH_NAME varchar 512 1536 utf8 utf8_general_ci varchar(512) NULL information_schema SPATIAL_REF_SYS AUTH_SRID int NULL NULL NULL NULL int(5) 3.0000 information_schema SPATIAL_REF_SYS SRTEXT varchar 2048 6144 utf8 utf8_general_ci varchar(2048) +3.0000 information_schema SQL_FUNCTIONS FUNCTION varchar 64 192 utf8 utf8_general_ci varchar(64) 3.0000 information_schema STATISTICS TABLE_CATALOG varchar 512 1536 utf8 utf8_general_ci varchar(512) 3.0000 information_schema STATISTICS TABLE_SCHEMA varchar 64 192 utf8 utf8_general_ci varchar(64) 3.0000 information_schema STATISTICS TABLE_NAME varchar 64 192 utf8 utf8_general_ci varchar(64) diff --git a/mysql-test/suite/funcs_1/r/is_tables_is.result b/mysql-test/suite/funcs_1/r/is_tables_is.result index 502b165cde3..aca57297574 100644 --- a/mysql-test/suite/funcs_1/r/is_tables_is.result +++ b/mysql-test/suite/funcs_1/r/is_tables_is.result @@ -405,6 +405,29 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME KEYWORDS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME KEY_CACHES TABLE_TYPE SYSTEM VIEW ENGINE MEMORY @@ -704,6 +727,29 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME SQL_FUNCTIONS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME STATISTICS TABLE_TYPE SYSTEM VIEW ENGINE MEMORY @@ -1364,6 +1410,29 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME KEYWORDS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME KEY_CACHES TABLE_TYPE SYSTEM VIEW ENGINE MEMORY @@ -1663,6 +1732,29 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME SQL_FUNCTIONS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME STATISTICS TABLE_TYPE SYSTEM VIEW ENGINE MEMORY diff --git a/mysql-test/suite/funcs_1/r/is_tables_is_embedded.result b/mysql-test/suite/funcs_1/r/is_tables_is_embedded.result index 502b165cde3..aca57297574 100644 --- a/mysql-test/suite/funcs_1/r/is_tables_is_embedded.result +++ b/mysql-test/suite/funcs_1/r/is_tables_is_embedded.result @@ -405,6 +405,29 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME KEYWORDS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME KEY_CACHES TABLE_TYPE SYSTEM VIEW ENGINE MEMORY @@ -704,6 +727,29 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME SQL_FUNCTIONS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME STATISTICS TABLE_TYPE SYSTEM VIEW ENGINE MEMORY @@ -1364,6 +1410,29 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME KEYWORDS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME KEY_CACHES TABLE_TYPE SYSTEM VIEW ENGINE MEMORY @@ -1663,6 +1732,29 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME SQL_FUNCTIONS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME STATISTICS TABLE_TYPE SYSTEM VIEW ENGINE MEMORY diff --git a/sql/handler.h b/sql/handler.h index b3617d7a47a..542b91c570d 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -741,6 +741,7 @@ enum enum_schema_tables SCH_FILES, SCH_GLOBAL_STATUS, SCH_GLOBAL_VARIABLES, + SCH_KEYWORDS, SCH_KEY_CACHES, SCH_KEY_COLUMN_USAGE, SCH_OPEN_TABLES, @@ -756,6 +757,7 @@ enum enum_schema_tables SCH_SESSION_STATUS, SCH_SESSION_VARIABLES, SCH_STATISTICS, + SCH_SQL_FUNCTIONS, SCH_SYSTEM_VARIABLES, SCH_TABLES, SCH_TABLESPACES, diff --git a/sql/item_create.cc b/sql/item_create.cc index 4bfb2615fbe..a3c51d1949e 100644 --- a/sql/item_create.cc +++ b/sql/item_create.cc @@ -6745,13 +6745,6 @@ Create_func_year_week::create_native(THD *thd, LEX_STRING name, return func; } - -struct Native_func_registry -{ - LEX_STRING name; - Create_func *builder; -}; - #define BUILDER(F) & F::s_singleton #ifdef HAVE_SPATIAL @@ -6771,7 +6764,7 @@ struct Native_func_registry - keep 1 line per entry, it makes grep | sort easier */ -static Native_func_registry func_array[] = +Native_func_registry func_array[] = { { { C_STRING_WITH_LEN("ABS") }, BUILDER(Create_func_abs)}, { { C_STRING_WITH_LEN("ACOS") }, BUILDER(Create_func_acos)}, @@ -7113,6 +7106,8 @@ static Native_func_registry func_array[] = { {0, 0}, NULL} }; +size_t func_array_length= sizeof(func_array) / sizeof(Native_func_registry) - 1; + static HASH native_functions_hash; extern "C" uchar* diff --git a/sql/item_create.h b/sql/item_create.h index 97dc594b11c..3ce731e2d69 100644 --- a/sql/item_create.h +++ b/sql/item_create.h @@ -183,6 +183,12 @@ Item *create_temporal_literal(THD *thd, const String *str, type, send_error); } +struct Native_func_registry +{ + LEX_STRING name; + Create_func *builder; +}; + int item_create_init(); void item_create_cleanup(); diff --git a/sql/lex.h b/sql/lex.h index ca7c6635329..c0cb9adc236 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -45,7 +45,7 @@ SYM_GROUP sym_group_rtree= {"RTree keys", "HAVE_RTREE_KEYS"}; lists */ -static SYMBOL symbols[] = { +SYMBOL symbols[] = { { "&&", SYM(AND_AND_SYM)}, { "<=", SYM(LE)}, { "<>", SYM(NE)}, @@ -687,7 +687,7 @@ static SYMBOL symbols[] = { }; -static SYMBOL sql_functions[] = { +SYMBOL sql_functions[] = { { "ADDDATE", SYM(ADDDATE_SYM)}, { "BIT_AND", SYM(BIT_AND)}, { "BIT_OR", SYM(BIT_OR)}, @@ -732,4 +732,7 @@ static SYMBOL sql_functions[] = { { "VAR_SAMP", SYM(VAR_SAMP_SYM)}, }; +size_t symbols_length= sizeof(symbols) / sizeof(SYMBOL); +size_t sql_functions_length= sizeof(sql_functions) / sizeof(SYMBOL); + #endif /* LEX_INCLUDED */ diff --git a/sql/sql_show.cc b/sql/sql_show.cc index b5622497a06..f8f784bbbad 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -62,6 +62,19 @@ #ifdef WITH_PARTITION_STORAGE_ENGINE #include "ha_partition.h" #endif + +#include "lex_symbol.h" +#define KEYWORD_SIZE 64 + +extern SYMBOL symbols[]; +extern size_t symbols_length; + +extern SYMBOL sql_functions[]; +extern size_t sql_functions_length; + +extern Native_func_registry func_array[]; +extern size_t func_array_length; + enum enum_i_s_events_fields { ISE_EVENT_CATALOG= 0, @@ -7556,6 +7569,60 @@ int fill_variables(THD *thd, TABLE_LIST *tables, COND *cond) DBUG_RETURN(res); } +int add_symbol_to_table(const char* name, TABLE* table){ + DBUG_ENTER("add_symbol_to_table"); + + uint length= strlen(name); + + // If you've added a new SQL keyword longer than KEYWORD_SIZE, + // please increase the defined max length + DBUG_ASSERT(length < KEYWORD_SIZE); + + restore_record(table, s->default_values); + table->field[0]->set_notnull(); + table->field[0]->store(name, length, + system_charset_info); + if (schema_table_store_record(table->in_use, table)) + DBUG_RETURN(1); + + DBUG_RETURN(0); +} + +int fill_i_s_keywords(THD *thd, TABLE_LIST *tables, COND *cond) +{ + DBUG_ENTER("fill_i_s_keywords"); + + TABLE *table= tables->table; + + for (uint i= 0; i < symbols_length; i++){ + const char *name= symbols[i].name; + if (add_symbol_to_table(name, table)) + DBUG_RETURN(1); + } + + DBUG_RETURN(0); +} + +int fill_i_s_sql_functions(THD *thd, TABLE_LIST *tables, COND *cond) { + DBUG_ENTER("fill_i_s_sql_functions"); + + TABLE *table= tables->table; + + for (uint i= 0; i < sql_functions_length; i++){ + const char *name= sql_functions[i].name; + if (add_symbol_to_table(name, table)) + DBUG_RETURN(1); + } + + for (uint i= 0; i < func_array_length; i++){ + const char *name= func_array[i].name.str; + if (add_symbol_to_table(name, table)) + DBUG_RETURN(1); + } + + DBUG_RETURN(0); +} + int fill_status(THD *thd, TABLE_LIST *tables, COND *cond) { @@ -8763,6 +8830,18 @@ ST_FIELD_INFO enabled_roles_fields_info[]= {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} }; +ST_FIELD_INFO keywords_field_info[]= +{ + {"WORD", KEYWORD_SIZE, MYSQL_TYPE_STRING, 0, MY_I_S_MAYBE_NULL, 0, SKIP_OPEN_TABLE}, + {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} +}; + +ST_FIELD_INFO sql_functions_field_info[]= +{ + {"FUNCTION", KEYWORD_SIZE, MYSQL_TYPE_STRING, 0, MY_I_S_MAYBE_NULL, 0, SKIP_OPEN_TABLE}, + {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} +}; + ST_FIELD_INFO engines_fields_info[]= { @@ -9465,6 +9544,8 @@ ST_SCHEMA_TABLE schema_tables[]= fill_status, make_old_format, 0, 0, -1, 0, 0}, {"GLOBAL_VARIABLES", variables_fields_info, 0, fill_variables, make_old_format, 0, 0, -1, 0, 0}, + {"KEYWORDS", keywords_field_info, 0, + fill_i_s_keywords, 0, 0, -1, -1, 0, 0}, {"KEY_CACHES", keycache_fields_info, 0, fill_key_cache_tables, 0, 0, -1,-1, 0, 0}, {"KEY_COLUMN_USAGE", key_column_usage_fields_info, 0, @@ -9500,6 +9581,8 @@ ST_SCHEMA_TABLE schema_tables[]= {"STATISTICS", stat_fields_info, 0, get_all_tables, make_old_format, get_schema_stat_record, 1, 2, 0, OPEN_TABLE_ONLY|OPTIMIZE_I_S_TABLE}, + {"SQL_FUNCTIONS", sql_functions_field_info, 0, + fill_i_s_sql_functions, 0, 0, -1, -1, 0, 0}, {"SYSTEM_VARIABLES", sysvars_fields_info, 0, fill_sysvars, make_old_format, 0, 0, -1, 0, 0}, {"TABLES", tables_fields_info, 0, From eb20c91b55e4b51be533314994b36bf9b24016f3 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 29 Jun 2021 16:31:28 +0300 Subject: [PATCH 14/15] MDEV-25969: Condition pushdown into derived table doesn't work if select list uses SP Consider a query of the form: select ... from (select item2 as COL1) as T where COL1=123 Condition pushdown into derived table will try to push "COL1=123" condition down into table T. The process of pushdown involves "substituting" the item, that is, replacing Item_field("T.COL1") with its "producing item" item2. In order to use item2, one needs to clone it (call Item::build_clone). If the item is not cloneable (e.g. Item_func_sp is not), the pushdown process will fail and nothing at all will be pushed. Fixed by introducing transform_condition_or_part() which will try to apply the transformation for as many parts of condition as possible. The parts of condition that couldn't be transformed are dropped. --- mysql-test/r/derived_cond_pushdown.result | 146 ++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 70 +++++++++++ sql/sql_derived.cc | 79 +++++++++++- 3 files changed, 289 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 28532ae88a4..f5ef3c3a84d 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -10673,4 +10673,150 @@ Warnings: Note 1003 select `v2`.`a` AS `a`,`v2`.`f` AS `f`,`v2`.`g` AS `g` from `test`.`v2` where `v2`.`f` = `v2`.`a` and `v2`.`g` = `v2`.`a` drop view v1,v2; drop table t1; +# +# MDEV-25969: Condition pushdown into derived table doesn't work if select list uses SP +# +create function f1(a int) returns int DETERMINISTIC return (a+1); +create table t1 ( +pk int primary key, +a int, +b int, +key(a) +); +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 from t2 A, t2 B, t2 C; +insert into t1 select a,a,a from t3; +create view v1 as +select +t1.a as col1, +f1(t1.b) as col2 +from +t1; +create view v2 as +select +t1.a as col1, +f1(t1.b) as col2 +from +t1; +create view v3 as +select col2, col1 from v1 +union all +select col2, col1 from v2; +explain select * from v3 where col1=123; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 2 Using where +2 DERIVED t1 ref a a 5 const 1 +3 UNION t1 ref a a 5 const 1 +# This must use ref accesses for reading table t1, not full scans: +explain format=json +select * from v3 where col1=123 and col2=321; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "v3.col1 = 123 and v3.col2 = 321", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "table": { + "table_name": "t1", + "access_type": "ref", + "possible_keys": ["a"], + "key": "a", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["const"], + "rows": 1, + "filtered": 100 + } + } + }, + { + "query_block": { + "select_id": 3, + "table": { + "table_name": "t1", + "access_type": "ref", + "possible_keys": ["a"], + "key": "a", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["const"], + "rows": 1, + "filtered": 100 + } + } + } + ] + } + } + } + } + } +} +drop function f1; +drop view v1,v2,v3; +drop table t1, t2,t3; +# +# Another testcase, with pushdown through GROUP BY +# +create table t1 (a int, b int); +insert into t1 values (1,1),(2,2),(3,3); +create function f1(a int) returns int DETERMINISTIC return (a+1); +create view v2(a, a2, s) as +select a, f1(a), sum(b) from t1 group by a, f1(a); +# Here, +# "(s+1) > 10" will be pushed into HAVING +# "a > 1" will be pushed all the way to the table scan on t1 +# "a2>123" will be pushed into HAVING (as it refers to an SP call which +# prevents pushing it to the WHERE) +explain format=json +select * from v2 where (s+1) > 10 AND a > 1 and a2>123; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "v2.s + 1 > 10 and v2.a > 1 and v2.a2 > 123", + "materialized": { + "query_block": { + "select_id": 2, + "having_condition": "s + 1 > 10 and a2 > 123", + "filesort": { + "sort_key": "t1.a, f1(t1.a)", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "t1.a > 1" + } + } + } + } + } + } + } +} +drop view v2; +drop function f1; +drop table t1; # End of 10.2 tests diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index 58f38ac1e5a..9b7658a725e 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -2237,4 +2237,74 @@ eval explain extended $q2; drop view v1,v2; drop table t1; +--echo # +--echo # MDEV-25969: Condition pushdown into derived table doesn't work if select list uses SP +--echo # +create function f1(a int) returns int DETERMINISTIC return (a+1); + +create table t1 ( + pk int primary key, + a int, + b int, + key(a) +); + +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); + +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 from t2 A, t2 B, t2 C; + +insert into t1 select a,a,a from t3; + +create view v1 as +select + t1.a as col1, + f1(t1.b) as col2 +from + t1; + +create view v2 as +select + t1.a as col1, + f1(t1.b) as col2 +from + t1; +create view v3 as +select col2, col1 from v1 +union all +select col2, col1 from v2; + +explain select * from v3 where col1=123; + +--echo # This must use ref accesses for reading table t1, not full scans: +explain format=json +select * from v3 where col1=123 and col2=321; + +drop function f1; +drop view v1,v2,v3; +drop table t1, t2,t3; + +--echo # +--echo # Another testcase, with pushdown through GROUP BY +--echo # +create table t1 (a int, b int); +insert into t1 values (1,1),(2,2),(3,3); + +create function f1(a int) returns int DETERMINISTIC return (a+1); + +create view v2(a, a2, s) as +select a, f1(a), sum(b) from t1 group by a, f1(a); + +--echo # Here, +--echo # "(s+1) > 10" will be pushed into HAVING +--echo # "a > 1" will be pushed all the way to the table scan on t1 +--echo # "a2>123" will be pushed into HAVING (as it refers to an SP call which +--echo # prevents pushing it to the WHERE) +explain format=json +select * from v2 where (s+1) > 10 AND a > 1 and a2>123; + +drop view v2; +drop function f1; +drop table t1; --echo # End of 10.2 tests diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index 632baf4bc5b..30bffeb7581 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -1192,6 +1192,68 @@ bool mysql_derived_reinit(THD *thd, LEX *lex, TABLE_LIST *derived) } +/* + @brief + Given condition cond and transformer+argument, try transforming as many + conjuncts as possible. + + @detail + The motivation of this function is to convert the condition that's being + pushed into a WHERE clause with derived_field_transformer_for_where or + with derived_grouping_field_transformer_for_where. + The transformer may fail for some sub-condition, in this case we want to + convert the most restrictive part of the condition that can be pushed. + + This function only does it for top-level AND: conjuncts that could not be + converted are dropped. + + @return + Converted condition, or NULL if nothing could be converted +*/ + +static +Item *transform_condition_or_part(THD *thd, + Item *cond, + Item_transformer transformer, + uchar *arg) +{ + if (cond->type() != Item::COND_ITEM || + ((Item_cond*) cond)->functype() != Item_func::COND_AND_FUNC) + { + Item *new_item= cond->transform(thd, transformer, arg); + // Indicate that the condition is not pushable + if (!new_item) + cond->clear_extraction_flag(); + return new_item; + } + + List_iterator li(*((Item_cond*) cond)->argument_list()); + Item *item; + while ((item=li++)) + { + Item *new_item= item->transform(thd, transformer, arg); + if (!new_item) + { + // Indicate that the condition is not pushable + item->clear_extraction_flag(); + li.remove(); + } + else + li.replace(new_item); + } + + switch (((Item_cond*) cond)->argument_list()->elements) + { + case 0: + return NULL; + case 1: + return ((Item_cond*) cond)->argument_list()->head(); + default: + return cond; + } +} + + /** @brief Extract the condition depended on derived table/view and pushed it there @@ -1287,9 +1349,11 @@ bool pushdown_cond_for_derived(THD *thd, Item *cond, TABLE_LIST *derived) if (!sl->join->group_list && !sl->with_sum_func) { /* extracted_cond_copy is pushed into where of sl */ - extracted_cond_copy= extracted_cond_copy->transform(thd, - &Item::derived_field_transformer_for_where, - (uchar*) sl); + extracted_cond_copy= + transform_condition_or_part(thd, + extracted_cond_copy, + &Item::derived_field_transformer_for_where, + (uchar*)sl); if (extracted_cond_copy) { extracted_cond_copy->walk( @@ -1316,9 +1380,12 @@ bool pushdown_cond_for_derived(THD *thd, Item *cond, TABLE_LIST *derived) pushed into the where clause of sl to make them usable in the new context */ if (cond_over_grouping_fields) - cond_over_grouping_fields= cond_over_grouping_fields->transform(thd, - &Item::derived_grouping_field_transformer_for_where, - (uchar*) sl); + { + cond_over_grouping_fields= + transform_condition_or_part(thd, cond_over_grouping_fields, + &Item::derived_grouping_field_transformer_for_where, + (uchar*) sl); + } if (cond_over_grouping_fields) { From 4a6e2d343745c11086c05f0041a8267591bb073c Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 30 Jun 2021 16:43:43 +0300 Subject: [PATCH 15/15] Post-merge fix: update derived_cond_pushdown.result --- mysql-test/main/derived_cond_pushdown.result | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result index b886db20eed..d2c116913f4 100644 --- a/mysql-test/main/derived_cond_pushdown.result +++ b/mysql-test/main/derived_cond_pushdown.result @@ -10764,6 +10764,7 @@ EXPLAIN { "query_block": { "select_id": 3, + "operation": "UNION", "table": { "table_name": "t1", "access_type": "ref",