From 7dc51833172d4d0a2e7f218a1fa6380d5bee35fe Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 2 Jan 2018 13:40:05 +1100 Subject: [PATCH 001/111] wsrep_sst_rsync: read correct configuration sections 4bb49d84a9df had a single option, --mysqld to obtain the required innodb_log_group_home_dir. 83664e21e4fb675 enumberated the groups when --mysqld was sufficient. --- scripts/wsrep_sst_rsync.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index c6e8a7ef4e1..de881459975 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -102,19 +102,7 @@ fi WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} # if WSREP_LOG_DIR env. variable is not set, try to get it from my.cnf if [ -z "$WSREP_LOG_DIR" ]; then - WSREP_LOG_DIR=$(parse_cnf mariadb-10.0 innodb_log_group_home_dir "") -fi -if [ -z "$WSREP_LOG_DIR" ]; then - WSREP_LOG_DIR=$(parse_cnf mysqld innodb_log_group_home_dir "") -fi -if [ -z "$WSREP_LOG_DIR" ]; then - WSREP_LOG_DIR=$(parse_cnf server innodb_log_group_home_dir "") -fi -if [ -z "$WSREP_LOG_DIR" ]; then - WSREP_LOG_DIR=$(parse_cnf mariadb innodb_log_group_home_dir "") -fi -if [ -z "$WSREP_LOG_DIR" ]; then - WSREP_LOG_DIR=$(parse_cnf mysqld-10.0 innodb_log_group_home_dir "") + WSREP_LOG_DIR=$(parse_cnf --mysqld innodb_log_group_home_dir "") fi if [ -n "$WSREP_LOG_DIR" ]; then From 67f3fb1bc54682a29ee3b407f60b798780db698f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 2 Jan 2018 14:26:48 +1100 Subject: [PATCH 002/111] galera_recovery: misses reading default configuration groups --- scripts/galera_recovery.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/galera_recovery.sh b/scripts/galera_recovery.sh index de2e653c497..f6a06a7e8e0 100644 --- a/scripts/galera_recovery.sh +++ b/scripts/galera_recovery.sh @@ -107,8 +107,7 @@ else log "WSREP: mktemp failed" fi -parse_arguments `$print_defaults $cmdline_args --loose-verbose \ - mariadb mariadb_safe mysqld mysqld_safe safe_mysqld galera` +parse_arguments `$print_defaults $cmdline_args --loose-verbose --mysqld` # Perform wsrep position recovery if wsrep_on=1, skip otherwise. if [ "$wsrep_on" -eq 1 ]; then From 1adc382c2fb4300059d130521a24a1dab141531e Mon Sep 17 00:00:00 2001 From: benrubson Date: Mon, 12 Feb 2018 22:08:57 +0100 Subject: [PATCH 003/111] Use stunnel during rsync SST if available --- scripts/wsrep_sst_rsync.sh | 67 ++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 3d76d1780e2..bb2f3fa7d64 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -41,6 +41,8 @@ cleanup_joiner() kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || \ : rm -rf "$RSYNC_CONF" + rm -f "$STUNNEL_CONF" + rm -f "$STUNNEL_PID" rm -rf "$MAGIC_FILE" rm -rf "$RSYNC_PID" wsrep_log_info "Joiner cleanup done." @@ -75,7 +77,7 @@ check_pid_and_port() local port_info=$(lsof -i :$rsync_port -Pn 2>/dev/null | \ grep "(LISTEN)") local is_rsync=$(echo $port_info | \ - grep -w '^rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null) + grep -wE '^(rsync|stunnel)[[:space:]]+'"$rsync_pid" 2>/dev/null) if [ -n "$port_info" -a -z "$is_rsync" ]; then wsrep_log_error "rsync daemon port '$rsync_port' has been taken" @@ -86,6 +88,12 @@ check_pid_and_port() [ $(cat $pid_file) -eq $rsync_pid ] } +STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf" +rm -f "$STUNNEL_CONF" + +STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" +rm -f "$STUNNEL_PID" + MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" rm -rf "$MAGIC_FILE" @@ -123,9 +131,28 @@ fi FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*') +SSTKEY=$(parse_cnf sst tkey "") +SSTCERT=$(parse_cnf sst tcert "") +STUNNEL="" +if [ -f "$SSTKEY" ] && [ -f "$SSTCERT" ] && wsrep_check_programs stunnel +then + STUNNEL="stunnel ${STUNNEL_CONF}" +fi + if [ "$WSREP_SST_OPT_ROLE" = "donor" ] then +cat << EOF > "$STUNNEL_CONF" +CApath = ${SSTCERT%/*} +foreground = yes +pid = $STUNNEL_PID +debug = warning +client = yes +connect = ${WSREP_SST_OPT_ADDR%/*} +TIMEOUTclose = 0 +verifyPeer = yes +EOF + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] then @@ -185,7 +212,8 @@ then # first, the normal directories, so that we can detect incompatible protocol RC=0 - rsync --owner --group --perms --links --specials \ + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT "${FILTER[@]}" "$WSREP_SST_OPT_DATA/" \ rsync://$WSREP_SST_OPT_ADDR >&2 || RC=$? @@ -208,7 +236,8 @@ then fi # second, we transfer InnoDB log files - rsync --owner --group --perms --links --specials \ + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '- **' "$WSREP_LOG_DIR/" \ rsync://$WSREP_SST_OPT_ADDR-log_dir >&2 || RC=$? @@ -227,7 +256,8 @@ then find . -maxdepth 1 -mindepth 1 -type d -not -name "lost+found" \ -print0 | xargs -I{} -0 -P $count \ - rsync --owner --group --perms --links --specials \ + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ --ignore-times --inplace --recursive --delete --quiet \ $WHOLE_FILE_OPT --exclude '*/ib_logfile*' "$WSREP_SST_OPT_DATA"/{}/ \ rsync://$WSREP_SST_OPT_ADDR/{} >&2 || RC=$? @@ -250,7 +280,8 @@ then echo "continue" # now server can resume updating data echo "$STATE" > "$MAGIC_FILE" - rsync --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR echo "done $STATE" @@ -308,8 +339,30 @@ EOF # listen at all interfaces (for firewalled setups) readonly RSYNC_PORT=${WSREP_SST_OPT_PORT:-4444} - rsync --daemon --no-detach --port $RSYNC_PORT --config "$RSYNC_CONF" & - RSYNC_REAL_PID=$! + +cat << EOF > "$STUNNEL_CONF" +key = $SSTKEY +cert = $SSTCERT +foreground = yes +pid = $STUNNEL_PID +debug = warning +client = no +[rsync] +accept = $RSYNC_PORT +exec = $(which rsync) +execargs = rsync --server --daemon --config=$RSYNC_CONF . +EOF + + if [ -z "$STUNNEL" ] + then + # listen at all interfaces (for firewalled setups) + rsync --daemon --no-detach --port $RSYNC_PORT --config "$RSYNC_CONF" & + RSYNC_REAL_PID=$! + else + stunnel "$STUNNEL_CONF" & + RSYNC_REAL_PID=$! + RSYNC_PID=$STUNNEL_PID + fi until check_pid_and_port $RSYNC_PID $RSYNC_REAL_PID $RSYNC_PORT do From 46d5e1f2fddbb4589a8763508cea8589941a2b2f Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 26 Jul 2018 15:42:06 +0200 Subject: [PATCH 004/111] MDEV-10754 wsrep_sst_rsync does not support innodb_data_home_dir If innodb_data_home_dir path specified in the configuration file then rsync IST/SST will fail because the wsrep_sst_rsync.sh script does not read this parameter from the configuration file and then tries to find the data files in the default directory. To fix this error, we need to add reading of the innodb_data_home_dir configuration parameter to the rsync-related SST script. --- .../galera/r/galera_sst_rsync_data_dir.result | 288 ++++++++++++++++++ .../galera/t/galera_sst_rsync_data_dir.cnf | 11 + .../galera/t/galera_sst_rsync_data_dir.test | 17 ++ scripts/wsrep_sst_rsync.sh | 9 +- 4 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result b/mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result new file mode 100644 index 00000000000..ff85a7d6c0f --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result @@ -0,0 +1,288 @@ +connection node_1; +connection node_2; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +Performing State Transfer on a server that starts from a clean var directory +This is accomplished by shutting down node #2 and removing its var directory before restarting it +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +Cleaning var directory ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_clean_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Killing server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Performing --wsrep-recover ... +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_kill_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf new file mode 100644 index 00000000000..d084740c8a6 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf @@ -0,0 +1,11 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=rsync +innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test new file mode 100644 index 00000000000..6220836c11d --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test @@ -0,0 +1,17 @@ +--source include/big_test.inc +--source include/galera_cluster.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--source suite/galera/include/galera_st_shutdown_slave.inc +--source suite/galera/include/galera_st_clean_slave.inc + +--source suite/galera/include/galera_st_kill_slave.inc +--source suite/galera/include/galera_st_kill_slave_ddl.inc +--source include/auto_increment_offset_restore.inc + +# cleanup temporary database files: +--remove_files_wildcard $MYSQL_TMP_DIR/rsync_test * +--rmdir $MYSQL_TMP_DIR/rsync_test diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 81120b66d5a..3659a60832c 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -113,6 +113,12 @@ else WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P) fi +INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""} +# if INNODB_DATA_HOME_DIR env. variable is not set, try to get it from my.cnf +if [ -z "INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf --mysqld innodb-data-home-dir '') +fi + # Old filter - include everything except selected # FILTER=(--exclude '*.err' --exclude '*.pid' --exclude '*.sock' \ # --exclude '*.conf' --exclude core --exclude 'galera.*' \ @@ -121,7 +127,8 @@ fi # New filter - exclude everything except dirs (schemas) and innodb files FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' - -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*') + -f '+ /wsrep_sst_binlog.tar' -f '+ $INNODB_DATA_HOME_DIR/ib_lru_dump' + -f '+ $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*') if [ "$WSREP_SST_OPT_ROLE" = "donor" ] then From 33110db0550f632408a71560d1636685c120efc5 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 31 Jul 2018 10:46:16 -0400 Subject: [PATCH 005/111] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 23e938e60e1..4acb69e9e07 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=61 +MYSQL_VERSION_PATCH=62 MYSQL_VERSION_EXTRA= From 1a7a018939e19a0037808489be0fd1680581aec3 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 18 Jul 2018 14:57:27 +0300 Subject: [PATCH 006/111] MDEV-16557 Remove INNOBASE_SHARE::idx_trans_tbl INNOBASE_SHARE: remove check_index_consistency(): iterates through keys and looks for InnoDB and .frm mismatches. ha_innobase::innobase_get_index(): now uses dict_table_get_index_on_name() dict_table_get_index_on_name(): uses strcmp() instead of innobase_casestrcmp() as we just need to know whether strings are equal or not --- storage/innobase/dict/dict0dict.cc | 2 +- storage/innobase/handler/ha_innodb.cc | 372 ++-------------------- storage/innobase/handler/ha_innodb.h | 43 --- storage/innobase/handler/handler0alter.cc | 6 - storage/innobase/row/row0merge.cc | 11 - 5 files changed, 32 insertions(+), 402 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 8dc29863367..36b19eeea46 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -6311,7 +6311,7 @@ dict_table_get_index_on_name( while (index != NULL) { if (index->is_committed() == committed - && innobase_strcasecmp(index->name, name) == 0) { + && strcmp(index->name, name) == 0) { return(index); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d8e805862f7..41cbca0d49d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -176,8 +176,6 @@ static int innobase_wsrep_set_checkpoint(handlerton* hton, const XID* xid); static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid); #endif /* WITH_WSREP */ -/** to protect innobase_open_files */ -static mysql_mutex_t innobase_share_mutex; /** to force correct commit order in binlog */ static ulong commit_threads = 0; static mysql_cond_t commit_cond; @@ -466,8 +464,6 @@ operation, we only do it every INNOBASE_WAKE_INTERVAL'th step. */ #define INNOBASE_WAKE_INTERVAL 32 static ulong innobase_active_counter = 0; -static hash_table_t* innobase_open_tables; - /** Allowed values of innodb_change_buffering */ static const char* innobase_change_buffering_values[IBUF_USE_COUNT] = { "none", /* IBUF_USE_NONE */ @@ -567,7 +563,6 @@ const struct _ft_vft_ext ft_vft_ext_result = {innobase_fts_get_version, /* Keys to register pthread mutexes/cond in the current file with performance schema */ -static mysql_pfs_key_t innobase_share_mutex_key; static mysql_pfs_key_t commit_cond_mutex_key; static mysql_pfs_key_t commit_cond_key; static mysql_pfs_key_t pending_checkpoint_mutex_key; @@ -576,7 +571,6 @@ static mysql_pfs_key_t thd_destructor_thread_key; static PSI_mutex_info all_pthread_mutexes[] = { PSI_KEY(commit_cond_mutex), PSI_KEY(pending_checkpoint_mutex), - PSI_KEY(innobase_share_mutex) }; static PSI_cond_info all_innodb_conds[] = { @@ -1230,23 +1224,6 @@ static SHOW_VAR innodb_status_variables[]= { {NullS, NullS, SHOW_LONG} }; -/************************************************************************//** -Handling the shared INNOBASE_SHARE structure that is needed to provide table -locking. Register the table name if it doesn't exist in the hash table. */ -static -INNOBASE_SHARE* -get_share( -/*======*/ - const char* table_name); /*!< in: table to lookup */ - -/************************************************************************//** -Free the shared object that was registered with get_share(). */ -static -void -free_share( -/*=======*/ - INNOBASE_SHARE* share); /*!< in/own: share to free */ - /*****************************************************************//** Frees a possible InnoDB trx object associated with the current THD. @return 0 or error number */ @@ -4413,10 +4390,6 @@ innobase_change_buffering_inited_ok: ibuf_max_size_update(srv_change_buffer_max_size); - innobase_open_tables = hash_create(200); - mysql_mutex_init(innobase_share_mutex_key, - &innobase_share_mutex, - MY_MUTEX_INIT_FAST); mysql_mutex_init(commit_cond_mutex_key, &commit_cond_m, MY_MUTEX_INIT_FAST); mysql_cond_init(commit_cond_key, &commit_cond, 0); @@ -4492,9 +4465,6 @@ innobase_end(handlerton*, ha_panic_function) } } - hash_table_free(innobase_open_tables); - innobase_open_tables = NULL; - st_my_thread_var* running = reinterpret_cast( my_atomic_loadptr_explicit( reinterpret_cast(&srv_running), @@ -4512,7 +4482,6 @@ innobase_end(handlerton*, ha_panic_function) innodb_shutdown(); innobase_space_shutdown(); - mysql_mutex_destroy(&innobase_share_mutex); mysql_mutex_destroy(&commit_cond_m); mysql_cond_destroy(&commit_cond); mysql_mutex_destroy(&pending_checkpoint_mutex); @@ -5753,6 +5722,7 @@ is consistent between KEY info from mysql and that from innodb index. @param[in] key_info Index info from mysql @param[in] index_info Index info from InnoDB @return true if all column types match. */ +static bool innobase_match_index_columns( const KEY* key_info, @@ -6042,90 +6012,34 @@ innobase_build_v_templ( s_templ->tb_name = table->s->table_name.str; } -/*******************************************************************//** -This function builds a translation table in INNOBASE_SHARE -structure for fast index location with mysql array number from its -table->key_info structure. This also provides the necessary translation -between the key order in mysql key_info and InnoDB ib_table->indexes if -they are not fully matched with each other. -Note we do not have any mutex protecting the translation table -building based on the assumption that there is no concurrent -index creation/drop and DMLs that requires index lookup. All table -handle will be closed before the index creation/drop. -@return true if index translation table built successfully */ -static -bool -innobase_build_index_translation( -/*=============================*/ - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - dict_table_t* ib_table,/*!< in: table in InnoDB data - dictionary */ - INNOBASE_SHARE* share) /*!< in/out: share structure - where index translation table - will be constructed in. */ +/** Check consistency between .frm indexes and InnoDB indexes. +@param[in] table table object formed from .frm +@param[in] ib_table InnoDB table definition +@retval true if not errors were found */ +static bool +check_index_consistency(const TABLE* table, const dict_table_t* ib_table) { - DBUG_ENTER("innobase_build_index_translation"); - - bool ret = true; - - mutex_enter(&dict_sys->mutex); - - ulint mysql_num_index = table->s->keys; - ulint ib_num_index = UT_LIST_GET_LEN(ib_table->indexes); - dict_index_t** index_mapping = share->idx_trans_tbl.index_mapping; + ulint mysql_num_index = table->s->keys; + ulint ib_num_index = UT_LIST_GET_LEN(ib_table->indexes); + bool ret = true; /* If there exists inconsistency between MySQL and InnoDB dictionary (metadata) information, the number of index defined in MySQL - could exceed that in InnoDB, do not build index translation - table in such case */ + could exceed that in InnoDB, return error */ if (ib_num_index < mysql_num_index) { ret = false; goto func_exit; } - /* If index entry count is non-zero, nothing has - changed since last update, directly return TRUE */ - if (share->idx_trans_tbl.index_count) { - /* Index entry count should still match mysql_num_index */ - ut_a(share->idx_trans_tbl.index_count == mysql_num_index); - goto func_exit; - } - - /* The number of index increased, rebuild the mapping table */ - if (mysql_num_index > share->idx_trans_tbl.array_size) { - - index_mapping = reinterpret_cast( - ut_realloc(index_mapping, - mysql_num_index * sizeof(*index_mapping))); - - if (index_mapping == NULL) { - /* Report an error if index_mapping continues to be - NULL and mysql_num_index is a non-zero value */ - sql_print_error("InnoDB: fail to allocate memory for " - "index translation table. Number of " - "Index: " ULINTPF - ", array size:" ULINTPF, - mysql_num_index, - share->idx_trans_tbl.array_size); - ret = false; - goto func_exit; - } - - share->idx_trans_tbl.array_size = mysql_num_index; - } - /* For each index in the mysql key_info array, fetch its corresponding InnoDB index pointer into index_mapping array. */ for (ulint count = 0; count < mysql_num_index; count++) { - /* Fetch index pointers into index_mapping according to mysql - index sequence */ - index_mapping[count] = dict_table_get_index_on_name( - ib_table, table->key_info[count].name); + const dict_index_t* index = dict_table_get_index_on_name( + ib_table, table->key_info[count].name); - if (index_mapping[count] == 0) { + if (index == NULL) { sql_print_error("Cannot find index %s in InnoDB" " index dictionary.", table->key_info[count].name); @@ -6136,7 +6050,7 @@ innobase_build_index_translation( /* Double check fetched index has the same column info as those in mysql key_info. */ if (!innobase_match_index_columns(&table->key_info[count], - index_mapping[count])) { + index)) { sql_print_error("Found index %s whose column info" " does not match that of MariaDB.", table->key_info[count].name); @@ -6145,51 +6059,10 @@ innobase_build_index_translation( } } - /* Successfully built the translation table */ - share->idx_trans_tbl.index_count = mysql_num_index; - func_exit: - if (!ret) { - /* Build translation table failed. */ - ut_free(index_mapping); - - share->idx_trans_tbl.array_size = 0; - share->idx_trans_tbl.index_count = 0; - index_mapping = NULL; - } - - share->idx_trans_tbl.index_mapping = index_mapping; - - mutex_exit(&dict_sys->mutex); - - DBUG_RETURN(ret); + return ret; } -/*******************************************************************//** -This function uses index translation table to quickly locate the -requested index structure. -Note we do not have mutex protection for the index translatoin table -access, it is based on the assumption that there is no concurrent -translation table rebuild (fter create/drop index) and DMLs that -require index lookup. -@return dict_index_t structure for requested index. NULL if -fail to locate the index structure. */ -static -dict_index_t* -innobase_index_lookup( -/*==================*/ - INNOBASE_SHARE* share, /*!< in: share structure for index - translation table. */ - uint keynr) /*!< in: index number for the requested - index */ -{ - if (share->idx_trans_tbl.index_mapping == NULL - || keynr >= share->idx_trans_tbl.index_count) { - return(NULL); - } - - return(share->idx_trans_tbl.index_mapping[keynr]); -} /********************************************************************//** Get the upper limit of the MySQL integral and floating-point type. @return maximum allowed value for the field */ @@ -6319,11 +6192,6 @@ ha_innobase::open(const char* name, int, uint) m_user_thd = NULL; - if (!(m_share = get_share(name))) { - - DBUG_RETURN(1); - } - /* Will be allocated if it is needed in ::update_row() */ m_upd_buf = NULL; m_upd_buf_size = 0; @@ -6347,7 +6215,6 @@ ha_innobase::open(const char* name, int, uint) norm_name); } no_such_table: - free_share(m_share); set_my_errno(ENOENT); DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); @@ -6430,7 +6297,6 @@ no_such_table: } if (!thd_tablespace_op(thd) && no_tablespace) { - free_share(m_share); set_my_errno(ENOENT); int ret_err = HA_ERR_NO_SUCH_TABLE; @@ -6506,9 +6372,10 @@ no_such_table: mutex_exit(&dict_sys->mutex); } - if (!innobase_build_index_translation(table, ib_table, m_share)) { - sql_print_error("Build InnoDB index translation table for" - " Table %s failed", name); + if (!check_index_consistency(table, ib_table)) { + sql_print_error("InnoDB indexes are inconsistent with what " + "defined in .frm for table %s", + name); } /* Allocate a buffer for a 'row reference'. A row reference is @@ -6604,9 +6471,6 @@ no_such_table: /* Index block size in InnoDB: used by MySQL in query optimization */ stats.block_size = UNIV_PAGE_SIZE; - /* Init table lock structure */ - thr_lock_data_init(&m_share->lock, &lock, NULL); - if (m_prebuilt->table != NULL) { /* We update the highest file format in the system table space, if this table has higher file format setting. */ @@ -6802,8 +6666,6 @@ ha_innobase::close() m_upd_buf_size = 0; } - free_share(m_share); - MONITOR_INC(MONITOR_TABLE_CLOSE); /* Tell InnoDB server that there might be work for @@ -9654,60 +9516,18 @@ ha_innobase::innobase_get_index( clustered index, even if it was internally generated by InnoDB */ { - KEY* key; + KEY* key = NULL; + dict_table_t* ib_table = m_prebuilt->table; dict_index_t* index; DBUG_ENTER("innobase_get_index"); if (keynr != MAX_KEY && table->s->keys > 0) { - - key = table->key_info + keynr; - - index = innobase_index_lookup(m_share, keynr); - - if (index != NULL) { - if (!key || ut_strcmp(index->name, key->name) != 0) { - ib::error() << " Index for key no " << keynr - << " mysql name " << (key ? key->name : "NULL") - << " InnoDB name " << index->name() - << " for table " << m_prebuilt->table->name.m_name; - - for(uint i=0; i < table->s->keys; i++) { - index = innobase_index_lookup(m_share, i); - key = table->key_info + keynr; - - if (index) { - ib::info() << " Index for key no " << keynr - << " mysql name " << (key ? key->name : "NULL") - << " InnoDB name " << index->name() - << " for table " << m_prebuilt->table->name.m_name; - } - } - - } - - ut_a(ut_strcmp(index->name, key->name) == 0); - } else { - /* Can't find index with keynr in the translation - table. Only print message if the index translation - table exists */ - if (m_share->idx_trans_tbl.index_mapping != NULL) { - sql_print_warning("InnoDB could not find" - " index %s key no %u for" - " table %s through its" - " index translation table", - key ? key->name : "NULL", - keynr, - m_prebuilt->table->name - .m_name); - } - - index = dict_table_get_index_on_name( - m_prebuilt->table, key->name); - } + key = &table->key_info[keynr]; + index = dict_table_get_index_on_name(ib_table, key->name); + ut_ad(index); } else { - key = 0; - index = dict_table_get_first_index(m_prebuilt->table); + index = dict_table_get_first_index(ib_table); } if (index == NULL) { @@ -9715,7 +9535,7 @@ ha_innobase::innobase_get_index( "InnoDB could not find key no %u with name %s" " from dict cache for table %s", keynr, key ? key->name : "NULL", - m_prebuilt->table->name.m_name); + ib_table->name.m_name); } DBUG_RETURN(index); @@ -14014,20 +13834,12 @@ innodb_set_buf_pool_size(ulonglong buf_pool_size) } /*********************************************************************//** -Calculates the key number used inside MySQL for an Innobase index. We will -first check the "index translation table" for a match of the index to get -the index number. If there does not exist an "index translation table", -or not able to find the index in the translation table, then we will fall back -to the traditional way of looping through dict_index_t list to find a -match. In this case, we have to take into account if we generated a -default clustered index for the table +Calculates the key number used inside MySQL for an Innobase index. @return the key number used inside MySQL */ static int innobase_get_mysql_key_number_for_index( /*====================================*/ - INNOBASE_SHARE* share, /*!< in: share structure for index - translation table. */ const TABLE* table, /*!< in: table in MySQL data dictionary */ dict_table_t* ib_table,/*!< in: table in InnoDB data @@ -14058,27 +13870,8 @@ innobase_get_mysql_key_number_for_index( return(i); } - /* If index translation table exists, we will first check - the index through index translation table for a match. */ - if (share->idx_trans_tbl.index_mapping != NULL) { - for (i = 0; i < share->idx_trans_tbl.index_count; i++) { - if (share->idx_trans_tbl.index_mapping[i] == index) { - return(i); - } - } - - /* Print an error message if we cannot find the index - in the "index translation table". */ - if (index->is_committed()) { - sql_print_error("Cannot find index %s in InnoDB index" - " translation table.", index->name()); - } - } - - /* If we do not have an "index translation table", or not able - to find the index in the translation table, we'll directly find - matching index with information from mysql TABLE structure and - InnoDB dict_index_t list */ + /* Directly find matching index with information from mysql TABLE + structure and InnoDB dict_index_t list */ for (i = 0; i < table->s->keys; i++) { ind = dict_table_get_index_on_name( ib_table, table->key_info[i].name); @@ -14439,11 +14232,6 @@ ha_innobase::info_low( for (i = 0; i < table->s->keys; i++) { ulong j; - /* We could get index quickly through internal - index mapping with the index translation table. - The identity of index (match up index name with - that of table->key_info[i]) is already verified in - innobase_get_index(). */ dict_index_t* index = innobase_get_index(i); if (index == NULL) { @@ -14551,7 +14339,7 @@ ha_innobase::info_low( if (err_index) { errkey = innobase_get_mysql_key_number_for_index( - m_share, table, ib_table, err_index); + table, ib_table, err_index); } else { errkey = (unsigned int) ( (m_prebuilt->trx->error_key_num @@ -16547,104 +16335,6 @@ innobase_show_status( /* Success */ return(false); } - -/************************************************************************//** -Handling the shared INNOBASE_SHARE structure that is needed to provide table -locking. Register the table name if it doesn't exist in the hash table. */ -static -INNOBASE_SHARE* -get_share( -/*======*/ - const char* table_name) -{ - INNOBASE_SHARE* share; - - mysql_mutex_lock(&innobase_share_mutex); - - ulint fold = ut_fold_string(table_name); - - HASH_SEARCH(table_name_hash, innobase_open_tables, fold, - INNOBASE_SHARE*, share, - ut_ad(share->use_count > 0), - !strcmp(share->table_name, table_name)); - - if (share == NULL) { - - uint length = (uint) strlen(table_name); - - /* TODO: invoke HASH_MIGRATE if innobase_open_tables - grows too big */ - - share = reinterpret_cast( - my_malloc(//PSI_INSTRUMENT_ME, - sizeof(*share) + length + 1, - MYF(MY_FAE | MY_ZEROFILL))); - - share->table_name = reinterpret_cast( - memcpy(share + 1, table_name, length + 1)); - - HASH_INSERT(INNOBASE_SHARE, table_name_hash, - innobase_open_tables, fold, share); - - thr_lock_init(&share->lock); - - /* Index translation table initialization */ - share->idx_trans_tbl.index_mapping = NULL; - share->idx_trans_tbl.index_count = 0; - share->idx_trans_tbl.array_size = 0; - } - - ++share->use_count; - - mysql_mutex_unlock(&innobase_share_mutex); - - return(share); -} - -/************************************************************************//** -Free the shared object that was registered with get_share(). */ -static -void -free_share( -/*=======*/ - INNOBASE_SHARE* share) /*!< in/own: table share to free */ -{ - mysql_mutex_lock(&innobase_share_mutex); - -#ifdef UNIV_DEBUG - INNOBASE_SHARE* share2; - ulint fold = ut_fold_string(share->table_name); - - HASH_SEARCH(table_name_hash, innobase_open_tables, fold, - INNOBASE_SHARE*, share2, - ut_ad(share->use_count > 0), - !strcmp(share->table_name, share2->table_name)); - - ut_a(share2 == share); -#endif /* UNIV_DEBUG */ - - --share->use_count; - - if (share->use_count == 0) { - ulint fold = ut_fold_string(share->table_name); - - HASH_DELETE(INNOBASE_SHARE, table_name_hash, - innobase_open_tables, fold, share); - - thr_lock_delete(&share->lock); - - /* Free any memory from index translation table */ - ut_free(share->idx_trans_tbl.index_mapping); - - my_free(share); - - /* TODO: invoke HASH_MIGRATE if innobase_open_tables - shrinks too much */ - } - - mysql_mutex_unlock(&innobase_share_mutex); -} - /*********************************************************************//** Returns number of THR_LOCK locks used for one instance of InnoDB table. InnoDB no longer relies on THR_LOCK locks so 0 value is returned. diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 1a47dab423f..ab1d3bbeac3 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -23,35 +23,6 @@ this program; if not, write to the Free Software Foundation, Inc., system clustered index when there is no primary key. */ extern const char innobase_index_reserve_name[]; -/* Structure defines translation table between mysql index and InnoDB -index structures */ -struct innodb_idx_translate_t { - - ulint index_count; /*!< number of valid index entries - in the index_mapping array */ - - ulint array_size; /*!< array size of index_mapping */ - - dict_index_t** index_mapping; /*!< index pointer array directly - maps to index in InnoDB from MySQL - array index */ -}; - -/** InnoDB table share */ -typedef struct st_innobase_share { - THR_LOCK lock; - const char* table_name; /*!< InnoDB table name */ - uint use_count; /*!< reference count, - incremented in get_share() - and decremented in - free_share() */ - void* table_name_hash; - /*!< hash table chain node */ - innodb_idx_translate_t - idx_trans_tbl; /*!< index translation table between - MySQL and InnoDB */ -} INNOBASE_SHARE; - /** Prebuilt structures in an InnoDB table handle used within MySQL */ struct row_prebuilt_t; @@ -492,9 +463,6 @@ protected: THR_LOCK_DATA lock; - /** information for MySQL table locking */ - INNOBASE_SHARE* m_share; - /** buffer used in updates */ uchar* m_upd_buf; @@ -631,17 +599,6 @@ trx_t* innobase_trx_allocate( MYSQL_THD thd); /*!< in: user thread handle */ -/** Match index columns between MySQL and InnoDB. -This function checks whether the index column information -is consistent between KEY info from mysql and that from innodb index. -@param[in] key_info Index info from mysql -@param[in] index_info Index info from InnoDB -@return true if all column types match. */ -bool -innobase_match_index_columns( - const KEY* key_info, - const dict_index_t* index_info); - /*********************************************************************//** This function checks each index name for a table against reserved system default primary index name 'GEN_CLUST_INDEX'. If a name diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 6e318a040ca..d4a5a43ca37 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -8883,12 +8883,6 @@ foreign_fail: log_append_on_checkpoint(NULL); - /* Invalidate the index translation table. In partitioned - tables, there is no share. */ - if (m_share) { - m_share->idx_trans_tbl.index_count = 0; - } - /* Tell the InnoDB server that there might be work for utility threads: */ diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 951c16394a7..348611217cb 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3791,17 +3791,6 @@ row_merge_drop_indexes( ut_ad(prev); ut_a(table->fts); fts_drop_index(table, index, trx); - /* Since - INNOBASE_SHARE::idx_trans_tbl - is shared between all open - ha_innobase handles to this - table, no thread should be - accessing this dict_index_t - object. Also, we should be - holding LOCK=SHARED MDL on the - table even after the MDL - upgrade timeout. */ - /* We can remove a DICT_FTS index from the cache, because we do not allow ADD FULLTEXT INDEX From 68ebfb31f215247d2fa08c8ed97a320191afc179 Mon Sep 17 00:00:00 2001 From: sachin Date: Tue, 5 Jun 2018 15:14:19 +0530 Subject: [PATCH 007/111] MDEV-16166 RBR breaks with HA_ERR_KEY_NOT_FOUND upon DELETE from table... with spatial index So the issue is since it is spatial index , at the time of searching index for key (Rows_log_event::find_row) we use wrong field image we use Field::itRAW while we should be using Field::itMBR --- mysql-test/suite/rpl/r/rpl_row_spatial.result | 14 ++++++++++++++ mysql-test/suite/rpl/t/rpl_row_spatial.test | 17 +++++++++++++++++ sql/key.cc | 3 ++- 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/rpl/r/rpl_row_spatial.result create mode 100644 mysql-test/suite/rpl/t/rpl_row_spatial.test diff --git a/mysql-test/suite/rpl/r/rpl_row_spatial.result b/mysql-test/suite/rpl/r/rpl_row_spatial.result new file mode 100644 index 00000000000..8f546fc479e --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_row_spatial.result @@ -0,0 +1,14 @@ +include/master-slave.inc +[connection master] +CREATE TABLE t1 (g POINT NOT NULL, SPATIAL INDEX(g)); +INSERT INTO t1 VALUES (ST_GEOMFROMTEXT('Point(1 1)')); +INSERT INTO t1 VALUES (ST_GEOMFROMTEXT('Point(2 1)')); +INSERT INTO t1 VALUES (ST_GEOMFROMTEXT('Point(1 2)')); +INSERT INTO t1 VALUES (ST_GEOMFROMTEXT('Point(2 2)')); +DELETE FROM t1 where MBREqual(g, ST_GEOMFROMTEXT('Point(1 2)')); +select count(*) from t1; +count(*) +3 +DELETE FROM t1; +drop table t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_row_spatial.test b/mysql-test/suite/rpl/t/rpl_row_spatial.test new file mode 100644 index 00000000000..00c3dd7c54d --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_spatial.test @@ -0,0 +1,17 @@ +--source include/have_binlog_format_row.inc +--source include/master-slave.inc + +CREATE TABLE t1 (g POINT NOT NULL, SPATIAL INDEX(g)); +INSERT INTO t1 VALUES (ST_GEOMFROMTEXT('Point(1 1)')); +INSERT INTO t1 VALUES (ST_GEOMFROMTEXT('Point(2 1)')); +INSERT INTO t1 VALUES (ST_GEOMFROMTEXT('Point(1 2)')); +INSERT INTO t1 VALUES (ST_GEOMFROMTEXT('Point(2 2)')); +DELETE FROM t1 where MBREqual(g, ST_GEOMFROMTEXT('Point(1 2)')); + +--sync_slave_with_master +select count(*) from t1; + +--connection master +DELETE FROM t1; +drop table t1; +--source include/rpl_end.inc diff --git a/sql/key.cc b/sql/key.cc index 700bf6a05a6..7e5a3309b10 100644 --- a/sql/key.cc +++ b/sql/key.cc @@ -145,7 +145,8 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, { key_length-= HA_KEY_BLOB_LENGTH; length= min(key_length, key_part->length); - uint bytes= key_part->field->get_key_image(to_key, length, Field::itRAW); + uint bytes= key_part->field->get_key_image(to_key, length, + key_info->flags & HA_SPATIAL ? Field::itMBR : Field::itRAW); if (with_zerofill && bytes < length) bzero((char*) to_key + bytes, length - bytes); to_key+= HA_KEY_BLOB_LENGTH; From 1e37fa70bdab8749dd5ffd6b71ed53ca78671b10 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sat, 4 Aug 2018 03:06:27 +0300 Subject: [PATCH 008/111] Updated list of unstable tests for 10.1.35 release --- mysql-test/unstable-tests | 173 ++++++++++++++++++++++++-------------- 1 file changed, 111 insertions(+), 62 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 9b00579944c..c59b9725754 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,7 +23,7 @@ # ############################################################################## -# Based on 10.1 c22ab56f0d690feee471e173a3d95acb642cd6dc +# Based on 10.1 701f0b8e366f957e8256e4741ca48424c84b7234 main.alter_table : Modified in 10.1.34 main.alter_table_trans : MDEV-12084 - timeout @@ -31,43 +31,62 @@ main.analyze_stmt_slow_query_log : MDEV-12237 - Wrong result main.assign_key_cache : Added in 10.1.34 main.assign_key_cache_debug : Added in 10.1.34 main.auth_named_pipe : MDEV-14724 - System error 2 -main.connect : Modified in 10.1.33 -main.connect_debug : Added in 10.1.33 +main.auto_increment : Modified in 10.1.35 +main.bootstrap : Modified in 10.1.35 +main.connect_debug : Added in 10.0.36 +main.count_distinct2 : MDEV-11768 - timeout main.create_delayed : MDEV-10605 - failed with timeout main.create_or_replace : Modified in 10.1.34 -main.ctype_ucs : Modified in 10.1.33 -main.ctype_utf16le : MDEV-10675: timeout or extra warnings -main.ctype_utf8mb4 : Modified in 10.1.33 -main.grant : Modified in 10.1.34 -main.grant_not_windows : Added in 10.1.34 +main.ctype_binary : Modified in 10.1.35 +main.ctype_eucjpms : Modified in 10.1.35 +main.ctype_euckr : Modified in 10.1.35 +main.ctype_gbk : Modified in 10.1.35 +main.ctype_latin1 : Modified in 10.1.35 +main.ctype_ucs : Modified in 10.1.35 +main.ctype_ujis : Modified in 10.1.35 +main.ctype_utf16le : Modified in 10.1.35 +main.ctype_utf16 : MDEV-10675: timeout or extra warnings; modified in 10.1.35 +main.ctype_utf32 : Modified in 10.1.35 +main.ctype_utf8 : Modified in 10.1.35 +main.ctype_utf8mb4 : Modified in 10.1.35 +main.debug_sync : MDEV-10607 - internal error +main.derived : Modified in 10.1.35 +main.derived_opt : MDEV-11768 - timeout main.events_2 : MDEV-13277 - Server crash main.events_bugs : MDEV-12892 - Crash in fill_schema_processlist main.events_slowlog : MDEV-12821 - Wrong result main.events_restart : MDEV-12236 - Server shutdown problem -main.func_misc : Modified in 10.1.33 -main.func_str : Modified in 10.1.33 +main.gis : MDEV-13411 - wrong result on P8 +main.grant : Modified in 10.1.35 +main.grant_not_windows : Added in 10.1.34 +main.grant2 : Modified in 10.0.36 +main.having : Modified in 10.1.35 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout main.index_merge_innodb : MDEV-7142 - Wrong execution plan, timeout with valgrind main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure main.insert_select : Modified in 10.1.34 +main.join : Modified in 10.1.35 +main.join_cache : Modified in 10.1.35 +main.join_outer : Modified in 10.1.35 main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist main.limit : Modified in 10.1.34 +main.log_tables-big : MDEV-13408 - wrong result +main.max_statement_time : Modified in 10.1.35 main.mdev-504 : MDEV-10607 - sporadic "can't connect" -main.mdev375 : MDEV-10607 - sporadic "can't connect"; modified in 10.1.33 +main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" main.myisam : Modified in 10.1.34 -main.myisam_recover : Modified in 10.1.33 main.mysql : Modified in 10.1.34 main.mysql_client_test_nonblock : MDEV-15096 - exec failed main.mysql_cp932 : Modified in 10.1.34 main.mysql_upgrade_noengine : MDEV-14355 - Plugin is busy main.mysqldump : Modified in 10.1.34 +main.mysqlhotcopy_myisam : MDEV-10995 - test hangs on debug build main.mysqlslap : MDEV-11801 - timeout; modified in 10.1.34 main.mysqltest : MDEV-9269 - fails on Alpha main.olap : Modified in 10.1.34 main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan -main.parser : Modified in 10.1.33 main.partition_debug_sync : MDEV-15669 - Deadlock found when trying to get lock main.partition_innodb_plugin : MDEV-12901 - Valgrind warnings main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count; modified in 10.1.34 @@ -75,42 +94,48 @@ main.query_cache : MDEV-12895 - Wrong result main.query_cache_debug : MDEV-15281 - Resize or similar command in progress main.range_vs_index_merge_innodb : MDEV-15283 - Server has gone away main.read_only_innodb : Modified in 10.1.34 -main.rename : Modified in 10.1.34 +main.rename : Modified in 10.1.35 main.selectivity : Modified in 10.1.34 main.set_statement : MDEV-13183 - Wrong result main.show_explain : MDEV-10674 - sporadic failure main.sp : Modified in 10.1.34 -main.sp-destruct : Modified in 10.1.33 -main.sp-innodb : Modified in 10.1.33 +main.sp-innodb : Modified in 10.0.36 +main.sp_notembedded : MDEV-10607 - internal error main.sp-security : MDEV-10607 - sporadic "can't connect" -main.stat_tables_par_innodb : MDEV-14155 - Wrong rounding -main.statistics : Modified in 10.1.33 +main.stat_tables : Modified in 10.1.35 +main.stat_tables_par_innodb : MDEV-14155 - wrong rounding +main.statistics : Modified in 10.1.35 main.statistics_close : Added in 10.1.34 main.status : MDEV-8510 - sporadic wrong result -main.subselect4 : Modified in 10.1.33 +main.subselect : Modified in 10.1.35 main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.subselect_sj : Modified in 10.1.33 -main.subselect_sj2_mat : Modified in 10.1.34 -main.symlink-aria-11902 : MDEV-15098 - error 40 from storage engine -main.symlink-myisam-11902 : MDEV-15098 - error 40 from storage engine +main.subselect_sj : Modified in 10.0.36 +main.subselect_sj_mat : Modified in 10.1.35 +main.subselect_sj2_mat : Modified in 10.1.35 +main.subselect4 : Modified in 10.0.36 main.tc_heuristic_recover : MDEV-15200 - wrong error on mysqld_stub_cmd main.trigger : Modified in 10.1.34 main.type_blob : MDEV-15195 - Wrong result +main.type_datetime : MDEV-14322 - wrong result main.type_datetime_hires : MDEV-10687 - timeout -main.variables : Modified in 10.1.33 -main.view : Modified in 10.1.33 +main.union : Modified in 10.1.35 +main.xa : MDEV-11769 - lock wait timeout #---------------------------------------------------------------- archive.archive_bitfield : MDEV-11771 - Extra warning -archive.discover : MDEV-10510 - Table is marked as crashed +archive.archive_symlink : MDEV-12170 - unexpected error on rmdir +archive.discover : MDEV-10510 - Table is marked as crashed archive.mysqlhotcopy_archive : MDEV-14726 - Table is marked as crashed +archive-test_sql_discovery.discover : MDEV-16817 - Table marked as crashed + #---------------------------------------------------------------- binlog.binlog_commit_wait : MDEV-10150 - Error: too much time elapsed binlog.binlog_flush_binlogs_delete_domain : MDEV-14431 - Wrong error code binlog.binlog_killed : MDEV-12925 - Wrong result +binlog.binlog_tmp_table_row : Added in 10.1.35 binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- @@ -125,21 +150,7 @@ binlog_encryption.rpl_typeconv : MDEV-14362 - Lost connection to MySQL serve #---------------------------------------------------------------- -connect.jdbc : Included file modified in 10.1.33 -connect.jdbc_new : Included file modified in 10.1.33 -connect.jdbc_oracle : Included file modified in 10.1.33 -connect.jdbc_postgresql : Modified in 10.1.33 -connect.json_java_2 : Included file modified in 10.1.33 -connect.json_java_3 : Included file modified in 10.1.33 -connect.json_mongo_c : Included file modified in 10.1.33 -connect.json_udf : Modified in 10.1.33 -connect.mongo_c : Included file modified in 10.1.33 -connect.mongo_java_2 : Included file modified in 10.1.33 -connect.mongo_java_3 : Included file modified in 10.1.33 -connect.mongo_test : Modified in 10.1.33 connect.pivot : MDEV-14803 - failed to discover table -connect.tbl_thread : Modified in 10.1.33 -connect.vcol : Added in 10.1.33 connect.zip : MDEV-13884 - Wrong result #---------------------------------------------------------------- @@ -150,11 +161,11 @@ encryption.encrypt_and_grep : MDEV-13765 - Wrong result encryption.innodb-bad-key-change : Modified in 10.1.34 encryption.innodb-bad-key-change2 : Modified in 10.1.34 encryption.innodb-bad-key-change4 : Modified in 10.1.34 +encryption.innodb-checksum-algorithm : MDEV-16896 - Server crash encryption.innodb-compressed-blob : MDEV-14728 - Unable to get certificate; modified in 10.1.34 encryption.innodb-discard-import : Modified in 10.1.34 encryption.innodb-encryption-disable : Modified in 10.1.34 -encryption.innodb_encryption_discard_import : MDEV-16116 - Wrong result; modified in 10.1.33 -encryption.innodb_encryption_filekeys : Modified in 10.1.33 +encryption.innodb_encryption_discard_import : MDEV-16116 - Wrong result encryption.innodb_encryption-page-compression : MDEV-12630 - crash or assertion failure encryption.innodb_encryption_row_compressed : MDEV-16113 - Crash encryption.innodb_first_page : MDEV-10689 - Crash @@ -163,10 +174,10 @@ encryption.innodb-force-corrupt : Modified in 10.1.34 encryption.innodb_lotoftables : MDEV-16111 - Wrong result encryption.innodb-missing-key : Modified in 10.1.34 encryption.innodb-page_encryption : MDEV-10641 - mutex problem -encryption.innodb-read-only : MDEV-14728 - Unable to get certificate +encryption.innodb-read-only : MDEV-14728 - Unable to get certificate; MDEV-16563 - Crash on startup encryption.innodb-redo-badkey : MDEV-12898 - Server hang on startup; modified in 10.1.34 encryption.innodb-redo-nokeys : Modified in 10.1.34 -encryption.innodb-remove-encryption : MDEV-16493 - Timeout in wait condition; a dded in 10.1.33 +encryption.innodb-remove-encryption : MDEV-16493 - Timeout in wait condition encryption.innodb_scrub : MDEV-8139 - scrubbing tests need fixing encryption.innodb_scrub_background : MDEV-8139 - scrubbing tests need fixing encryption.innodb_scrub_compressed : MDEV-8139 - scrubbing tests need fixing @@ -175,9 +186,12 @@ encryption.innodb_scrub_compressed : MDEV-8139 - scrubbing tests need f engines/iuds.* : Not maintained in timely manner engines/funcs.* : Not maintained in timely manner +engines/rr_trx.* : MDEV-10998 - tests not maintained #---------------------------------------------------------------- +federated.assisted_discovery : Include file modified in 10.0.36 +federated.federated_bug_35333 : MDEV-13410 - Wrong result federated.federated_bug_585688 : MDEV-12907 - Valgrind, MDEV-14805 - server crash federated.federated_innodb : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips federated.federated_partition : MDEV-10417 - Fails on Mips @@ -186,11 +200,14 @@ federated.federatedx : MDEV-10617 - Wrong checksum, timeouts #---------------------------------------------------------------- +funcs_1.is_engines_federated : Include file modified in 10.0.36 +funcs_1.memory_views : MDEV-11773 - timeout funcs_1.processlist_val_no_prot : MDEV-11223 - Wrong result funcs_1.processlist_val_ps : MDEV-12175 - Wrong plan funcs_2.memory_charset : MDEV-10290 - Timeout funcs_2.myisam_charset : MDEV-11535 - Timeout +funcs_2/charset.* : MDEV-10999 - test not maintained #---------------------------------------------------------------- @@ -215,13 +232,18 @@ handler.ps : Added in 10.1.34 #---------------------------------------------------------------- -innodb.alter_partitioned_xa : Added in 10.1.33 +heap.heap_auto_increment : Modified in 10.1.35 + +#---------------------------------------------------------------- + +innodb.alter_partitioned_xa : Added in 10.0.36 innodb.binlog_consistent : MDEV-10618 - Server fails to start innodb.doublewrite : MDEV-12905 - Lost connection to MySQL server +innodb.group_commit_crash : MDEV-11770 - checksum mismatch +innodb.group_commit_crash_no_optimize_thread : MDEV-11770 - checksum mismatch innodb.innodb-64k-crash : MDEV-13872 - Failure and crash on startup innodb.innodb-alter : Added in 10.1.34 innodb.innodb-alter-debug : MDEV-13182 - InnoDB: adjusting FSP_SPACE_FLAGS -innodb.innodb-alter-nullable : Modified in 10.1.33 innodb.innodb-alter-table : MDEV-10619 - Testcase timeout innodb.innodb-blob : MDEV-12053 - Client crash; modified in 10.1.34 innodb.innodb_bug14147491 : MDEV-11808 - wrong error codes @@ -232,24 +254,32 @@ innodb.innodb_defragment_small : Modified in 10.1.34 innodb.innodb-fk : MDEV-13832 - Assertion failure on shutdown innodb.innodb_max_recordsize_64k : MDEV-15203 - wrong result innodb.innodb-mdev7046 : Modified in 10.1.34 +innodb.innodb_monitor : MDEV-10939 - Testcase timeout innodb.innodb-page_compression_default : Modified in 10.1.34 innodb.innodb-page_compression_lzma : MDEV-14353 - wrong result on Fedora 25 innodb.innodb-page_compression_snappy : Modified in 10.1.34 innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb_stats : MDEV-10682 - wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - wrong result +innodb.innodb-wl5522 : Modified in 10.1.35 innodb.innodb_zip_innochecksum2 : MDEV-13882 - Warning: difficult to find free blocks innodb.lock_deleted : Added in 10.1.34 innodb.log_file_size : MDEV-15668 - Not found pattern innodb.recovery_shutdown : MDEV-15671 - Warning: database page corruption innodb.rename_table : Added in 10.1.34 innodb.row_format_redundant : MDEV-15192 - Trying to access missing tablespace +innodb.sp_temp_table : MDEV-16647 - Could not remove temporary table innodb.table_definition_cache_debug : MDEV-14206 - Extra warning innodb.table_flags : MDEV-14363 - Operating system error number 2 +innodb.xa_recovery : MDEV-15279 - mysqld got exception #---------------------------------------------------------------- innodb_fts.basic : Added in 10.1.34 +innodb_fts.fts_kill_query : Added in 10.1.35 +innodb_fts.innodb-fts-fic : MDEV-14154 - Assertion failure +innodb_fts.innodb_fts_misc_debug : MDEV-14156 - Unexpected warning +innodb_fts.sync_ddl : Added in 10.1.35 #---------------------------------------------------------------- @@ -257,11 +287,10 @@ maria.alter : Modified in 10.1.34 maria.insert_select : MDEV-12757 - Timeout maria.insert_select-7314 : MDEV-16492 - Timeout maria.lock : Modified in 10.1.34 -maria.maria : MDEV-14430 - Wrong result; modified in 10.1.34 +maria.maria : MDEV-14430 - Wrong result; modified in 10.1.35 #---------------------------------------------------------------- -mariabackup.absolute_ibdata_paths : Added in 10.1.33 mariabackup.backup_ssl : Added in 10.1.34 mariabackup.incremental_encrypted : MDEV-15667 - Timeout mariabackup.mdev-14447 : MDEV-15201 - Timeout @@ -291,37 +320,43 @@ multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_h #---------------------------------------------------------------- -parts.partition_alter_innodb : Include file modified in 10.1.33 -parts.partition_alter_maria : Include file modified in 10.1.33 -parts.partition_alter_myisam : Include file modified in 10.1.33 +parts.alter_data_directory_innodb : Added in 10.1.35 parts.partition_alter2_2_maria : MDEV-14364 - Lost connection to MySQL server during query parts.partition_auto_increment_archive : MDEV-16491 - Table marked as crashed parts.partition_auto_increment_maria : MDEV-14430 - Wrong result -parts.partition_debug_innodb : MDEV-15095 - table does not exist +parts.partition_exch_qa_10 : MDEV-11765 - wrong result parts.partition_innodb_status_file : MDEV-12901 - Valgrind +parts.truncate_locked : Added in 10.1.35 #---------------------------------------------------------------- perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x -perfschema.hostcache_ipv4_max_con : Modified in 10.1.33 -perfschema.hostcache_ipv6_max_con : Modified in 10.1.33 +perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown perfschema.partition : Added in 10.1.34 perfschema.privilege_table_io : MDEV-13184 - Extra lines +perfschema.rpl_gtid_func : MDEV-16897 - Wrong result perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match perfschema.stage_mdl_global : MDEV-11803 - wrong result on slow builders +perfschema.stage_mdl_procedure : MDEV-11545 - Wrong result perfschema.stage_mdl_table : MDEV-12638 - Wrong result perfschema.threads_mysql : MDEV-10677 - sporadic wrong result +perfschema_stress.* : MDEV-10996 - tests not maintained + #---------------------------------------------------------------- plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url -plugins.processlist : Added in 10.1.34 +plugins.processlist : Added in 10.1.35 plugins.server_audit : MDEV-9562 - crashes on sol10-sparc; modified in 10.1.34 plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc #---------------------------------------------------------------- +roles.create_and_grant_role : MDEV-11772 - wrong result + +#---------------------------------------------------------------- + rpl.last_insert_id : MDEV-10625 - warnings in error log rpl.rename : Added in 10.1.34 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips @@ -332,6 +367,7 @@ rpl.rpl_colSize : MDEV-16112 - Server crash rpl.rpl_ddl : MDEV-10417 - Fails on Mips rpl.rpl_domain_id_filter_io_crash : MDEV-14357 - Wrong result rpl.rpl_domain_id_filter_restart : MDEV-10684 - Wrong result +rpl.rpl_drop_db_fail : MDEV-16898 - Slave fails to start rpl.rpl_gtid_basic : MDEV-10681 - server startup problem rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master rpl.rpl_gtid_delete_domain : MDEV-14463 - Timeout in include @@ -343,6 +379,7 @@ rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_id : MDEV-15197 - Wrong result +rpl.rpl_insert_id_pk : MDEV-16567 - Assertion failure rpl.rpl_insert_ignore : MDEV-14365 - Lost connection to MySQL server during query rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips rpl.rpl_mariadb_slave_capability : MDEV-11018 - sporadic wrong events in binlog @@ -350,15 +387,17 @@ rpl.rpl_mdev6020 : MDEV-10417 - Fails on Mips rpl.rpl_mixed_implicit_commit_binlog : Modified in 10.1.34 rpl.rpl_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_non_direct_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed +rpl.rpl_non_direct_row_mixing_engines : MDEV-16561 - Timeout in master pos wait rpl.rpl_non_direct_stm_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_parallel : MDEV-10653 - Timeouts rpl.rpl_parallel_mdev6589 : MDEV-12979 - Assertion failure rpl.rpl_parallel_multilevel2 : MDEV-14723 - Timeout -rpl.rpl_parallel_optimistic : MDEV-10511 - Timeout; modified in 10.1.34 +rpl.rpl_parallel_optimistic : MDEV-10511 - Timeout; modified in 10.1.35 rpl.rpl_parallel_retry : MDEV-11119 - Server crash rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips rpl.rpl_password_boundaries : MDEV-11534 - Slave IO warnings +rpl.rpl_row_basic_11bugs : MDEV-12171 - Server failed to start rpl.rpl_row_basic_2myisam : MDEV-13875 - command "diff_files" failed rpl.rpl_row_drop_create_temp_table : MDEV-14487 - Wrong result rpl.rpl_row_img_blobs : MDEV-13875 - command "diff_files" failed @@ -384,6 +423,8 @@ rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries rpl.sec_behind_master-5114 : MDEV-13878 - Wrong result +rpl/extra/rpl_tests.* : MDEV-10994 - tests not maintained + #---------------------------------------------------------------- spider.* : MDEV-9329 - tests are too memory-consuming @@ -393,6 +434,7 @@ spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't spider/bg.ha : MDEV-9329 - failures on s390x spider/bg.ha_part : MDEV-9329 - Fails on Ubuntu/s390x spider/bg.spider_fixes : MDEV-7098 -Mutex problem, MDEV-9329 - failures on s390x +spider/bg.spider_fixes_part : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.spider3_fixes : MDEV-12639 - Packets out of order spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x @@ -415,11 +457,7 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x sys_vars.keep_files_on_create_basic : MDEV-10676 - timeout sys_vars.log_slow_admin_statements_func : MDEV-12235 - Server crash -sys_vars.max_prepared_stmt_count_basic : Modified in 10.1.33 sys_vars.rpl_init_slave_func : MDEV-10149 - wrong results -sys_vars.sysvars_innodb : Opt file modified in 10.1.33 -sys_vars.sysvars_server_embedded : Opt file added in 10.1.33 -sys_vars.sysvars_server_notembedded : Opt file added in 10.1.33 sys_vars.thread_cache_size_func : MDEV-11775 - Wrong result sys_vars.wait_timeout_func : MDEV-12896 - Wrong result @@ -438,18 +476,29 @@ tokudb.hotindex-insert-0 : MDEV-15271 - Timeout tokudb.hotindex-insert-1 : MDEV-13870 - Lost connection to MySQL server tokudb.hotindex-update-0 : MDEV-15198 - Timeout tokudb.hotindex-update-1 : MDEV-12640 - Crash +tokudb.locks-select-update-1 : MDEV-13406 - Lock wait timeout tokudb.rows-32m-rand-insert : MDEV-12640 - Crash tokudb.rows-32m-seq-insert : MDEV-12640 - Crash tokudb.savepoint-5 : MDEV-15280 - Wrong result tokudb.type_datetime : MDEV-15193 - Wrong result +tokudb_backup.* : MDEV-11001 - tests don't work + tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output -tokudb_bugs.db917 : Modified in 10.1.33 +tokudb_bugs.frm_store : MDEV-12823 - Valgrind +tokudb_bugs.frm_store2 : MDEV-12823 - Valgrind +tokudb_bugs.frm_store3 : MDEV-12823 - Valgrind tokudb_bugs.xa : MDEV-11804 - Lock wait timeout +tokudb_rpl.* : MDEV-11001 - tests don't work +tokudb_sys_vars.* : MDEV-11001 - tests don't work +rpl-tokudb.* : MDEV-14354 - Tests fail with tcmalloc + #---------------------------------------------------------------- +unit.* : suite.pm modified in 10.1.35 + unit.lf : MDEV-12897 - Signal 11 thrown unit.ma_test_loghandler : MDEV-10638 - record read not ok unit.my_atomic : MDEV-15670 - Signal 11 thrown @@ -458,10 +507,10 @@ unit.my_atomic : MDEV-15670 - Signal 11 thrown vcol.not_supported : MDEV-10639 - Testcase timeout vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout +vcol.vcol_misc : MDEV-16651 - Wrong error message; modified in 10.1.35 #---------------------------------------------------------------- -wsrep.binlog_format : MDEV-11532 - WSREP has not yet prepared node wsrep.foreign_key : MDEV-14725 - WSREP has not yet prepared node wsrep.mdev_6832 : MDEV-14195 - Failure upon check-testcase wsrep.pool_of_threads : MDEV-12234 - Library problem on Power From 50c426200224a4527e84052aa2ab32be893f43f4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 4 Aug 2018 22:53:16 +0100 Subject: [PATCH 009/111] MDEV-16544 - crash in ha_sphinx::create() Use table_arg that was passed to the function, instead of dereferencing this->table, which is a NULL pointer. --- storage/sphinx/ha_sphinx.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/sphinx/ha_sphinx.cc b/storage/sphinx/ha_sphinx.cc index 3f6770b5d26..b496450c5d6 100644 --- a/storage/sphinx/ha_sphinx.cc +++ b/storage/sphinx/ha_sphinx.cc @@ -3440,7 +3440,7 @@ int ha_sphinx::create ( const char * name, TABLE * table_arg, HA_CREATE_INFO * ) if ( table_arg->s->keys!=1 || table_arg->key_info[0].user_defined_key_parts!=1 || - strcasecmp ( table_arg->key_info[0].key_part[0].field->field_name, table->field[2]->field_name ) ) + strcasecmp ( table_arg->key_info[0].key_part[0].field->field_name, table_arg->field[2]->field_name ) ) { my_snprintf ( sError, sizeof(sError), "%s: there must be an index on '%s' column", name, table_arg->field[2]->field_name ); From 9419908f3889e20d8edd299ca714fd779613cf5e Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 4 Aug 2018 05:57:06 +0530 Subject: [PATCH 010/111] MDEV-15433: Optimizer does not use group by optimization with distinct After the commit b76b69cd5fe634d8ddb9406aa2c82ef2a375b4d8 loose index scan for queries with DISTINCT stopped working. That is why that commit has to be reverted. Additionally this patch fixes the problem of MDEV-10880. --- mysql-test/r/bench_count_distinct.result | 2 +- mysql-test/r/distinct.result | 2 +- mysql-test/r/explain_json.result | 18 +++--- mysql-test/r/group_min_max.result | 63 ++++++++++++------- .../suite/vcol/r/vcol_select_innodb.result | 2 +- .../suite/vcol/r/vcol_select_myisam.result | 2 +- mysql-test/t/group_min_max.test | 27 ++++++++ sql/sql_select.cc | 10 ++- 8 files changed, 89 insertions(+), 37 deletions(-) diff --git a/mysql-test/r/bench_count_distinct.result b/mysql-test/r/bench_count_distinct.result index 79e12afd237..8b67e4be38a 100644 --- a/mysql-test/r/bench_count_distinct.result +++ b/mysql-test/r/bench_count_distinct.result @@ -5,7 +5,7 @@ count(distinct n) 100 explain extended select count(distinct n) from t1; id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE t1 index NULL n 4 NULL 200 100.00 Using index +1 SIMPLE t1 range NULL n 4 NULL 10 100.00 Using index for group-by Warnings: Note 1003 select count(distinct `test`.`t1`.`n`) AS `count(distinct n)` from `test`.`t1` drop table t1; diff --git a/mysql-test/r/distinct.result b/mysql-test/r/distinct.result index b5e8cefca69..d6e5a69e217 100644 --- a/mysql-test/r/distinct.result +++ b/mysql-test/r/distinct.result @@ -212,7 +212,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index NULL PRIMARY 4 NULL 1 Using index explain SELECT distinct a from t3 order by a desc limit 2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 index NULL a 5 NULL 2 Using index +1 SIMPLE t3 index NULL a 5 NULL 40 Using index explain SELECT distinct a,b from t3 order by a+1; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t3 ALL NULL NULL NULL NULL 204 Using temporary; Using filesort diff --git a/mysql-test/r/explain_json.result b/mysql-test/r/explain_json.result index 9e5515f3cfa..0abbda352b7 100644 --- a/mysql-test/r/explain_json.result +++ b/mysql-test/r/explain_json.result @@ -1030,10 +1030,10 @@ Table Op Msg_type Msg_text test.t1 analyze status Table is already up to date explain select count(distinct a1,a2,b) from t1 where (a2 >= 'b') and (b = 'a'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx_t1_2 147 NULL 128 Using where; Using index +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 17 Using where; Using index for group-by explain select count(distinct a1,a2,b,c) from t1 where (a2 >= 'b') and (b = 'a') and (c = 'i121'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx_t1_1 163 NULL 128 Using where; Using index +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 65 Using where; Using index for group-by (scanning) explain format=json select count(distinct a1,a2,b) from t1 where (a2 >= 'b') and (b = 'a'); EXPLAIN { @@ -1041,14 +1041,14 @@ EXPLAIN "select_id": 1, "table": { "table_name": "t1", - "access_type": "index", - "key": "idx_t1_2", + "access_type": "range", + "key": "idx_t1_1", "key_length": "147", "used_key_parts": ["a1", "a2", "b"], - "rows": 128, + "rows": 17, "filtered": 100, "attached_condition": "((t1.b = 'a') and (t1.a2 >= 'b'))", - "using_index": true + "using_index_for_group_by": true } } } @@ -1059,14 +1059,14 @@ EXPLAIN "select_id": 1, "table": { "table_name": "t1", - "access_type": "index", + "access_type": "range", "key": "idx_t1_1", "key_length": "163", "used_key_parts": ["a1", "a2", "b", "c"], - "rows": 128, + "rows": 65, "filtered": 100, "attached_condition": "((t1.b = 'a') and (t1.c = 'i121') and (t1.a2 >= 'b'))", - "using_index": true + "using_index_for_group_by": "scanning" } } } diff --git a/mysql-test/r/group_min_max.result b/mysql-test/r/group_min_max.result index cd7f1014ec0..ec3f4d9bf99 100644 --- a/mysql-test/r/group_min_max.result +++ b/mysql-test/r/group_min_max.result @@ -1707,13 +1707,13 @@ select a1,a2,b from t2 where (a1 > 'a') and (a2 > 'a') and (b = 'c') group by a1 a1 a2 b explain select distinct a1,a2,b from t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx_t1_2 147 NULL 128 Using index +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 17 Using index for group-by explain select distinct a1,a2,b from t1 where (a2 >= 'b') and (b = 'a'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx_t1_2 147 NULL 128 Using where; Using index +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 17 Using where; Using index for group-by explain extended select distinct a1,a2,b,c from t1 where (a2 >= 'b') and (b = 'a') and (c = 'i121'); id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE t1 index NULL idx_t1_1 163 NULL 128 100.00 Using where; Using index +1 SIMPLE t1 index NULL idx_t1_1 163 NULL 128 50.78 Using where; Using index Warnings: Note 1003 select distinct `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c` from `test`.`t1` where ((`test`.`t1`.`b` = 'a') and (`test`.`t1`.`c` = 'i121') and (`test`.`t1`.`a2` >= 'b')) explain select distinct a1,a2,b from t1 where (a1 > 'a') and (a2 > 'a') and (b = 'c'); @@ -1724,13 +1724,13 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index NULL idx_t1_2 147 NULL 128 Using where; Using index explain select distinct a1,a2,b from t2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 index NULL idx_t2_2 146 NULL # Using index +1 SIMPLE t2 range NULL idx_t2_1 146 NULL # Using index for group-by explain select distinct a1,a2,b from t2 where (a2 >= 'b') and (b = 'a'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 index NULL idx_t2_2 146 NULL # Using where; Using index +1 SIMPLE t2 range NULL idx_t2_1 146 NULL # Using where; Using index for group-by explain extended select distinct a1,a2,b,c from t2 where (a2 >= 'b') and (b = 'a') and (c = 'i121'); id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE t2 index NULL idx_t2_1 163 NULL 164 100.00 Using where; Using index +1 SIMPLE t2 index NULL idx_t2_1 163 NULL 164 50.61 Using where; Using index Warnings: Note 1003 select distinct `test`.`t2`.`a1` AS `a1`,`test`.`t2`.`a2` AS `a2`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t2` where ((`test`.`t2`.`b` = 'a') and (`test`.`t2`.`c` = 'i121') and (`test`.`t2`.`a2` >= 'b')) explain select distinct a1,a2,b from t2 where (a1 > 'a') and (a2 > 'a') and (b = 'c'); @@ -1855,7 +1855,7 @@ c e d e explain select distinct a1,a2,b from t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx_t1_2 147 NULL 128 Using index +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 17 Using index for group-by explain select distinct a1,a2,b from t1 where (a2 >= 'b') and (b = 'a') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range NULL idx_t1_1 147 NULL 17 Using where; Using index for group-by @@ -1870,7 +1870,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range NULL idx_t1_1 147 NULL 17 Using where; Using index for group-by; Using temporary; Using filesort explain select distinct a1,a2,b from t2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 index NULL idx_t2_2 146 NULL # Using index +1 SIMPLE t2 range NULL idx_t2_1 146 NULL # Using index for group-by explain select distinct a1,a2,b from t2 where (a2 >= 'b') and (b = 'a') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range NULL idx_t2_1 146 NULL # Using where; Using index for group-by @@ -1953,10 +1953,10 @@ b a explain select count(distinct a1,a2,b) from t1 where (a2 >= 'b') and (b = 'a'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx_t1_2 147 NULL 128 Using where; Using index +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 17 Using where; Using index for group-by explain select count(distinct a1,a2,b,c) from t1 where (a2 >= 'b') and (b = 'a') and (c = 'i121'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx_t1_1 163 NULL 128 Using where; Using index +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 65 Using where; Using index for group-by (scanning) explain extended select count(distinct a1,a2,b) from t1 where (a1 > 'a') and (a2 > 'a') and (b = 'c'); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 14 100.00 Using where; Using index for group-by @@ -2173,7 +2173,7 @@ c d explain select distinct a1 from t1 where a2 = 'b'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx_t1_2 147 NULL 128 Using where; Using index +1 SIMPLE t1 range NULL idx_t1_1 130 NULL 5 Using where; Using index for group-by select distinct a1 from t1 where a2 = 'b'; a1 a @@ -2283,7 +2283,7 @@ INSERT INTO t1 (a) VALUES ('SOUTH EAST'), ('SOUTH WEST'), ('WESTERN'); EXPLAIN SELECT DISTINCT a,a FROM t1 ORDER BY a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL a 66 NULL 11 Using index +1 SIMPLE t1 range NULL a 66 NULL 6 Using index for group-by SELECT DISTINCT a,a FROM t1 ORDER BY a; a a @@ -2499,7 +2499,7 @@ INSERT INTO t1 VALUES (4), (2), (1), (2), (2), (4), (1), (4); EXPLAIN SELECT DISTINCT(a) FROM t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx 5 NULL 16 Using index +1 SIMPLE t1 range NULL idx 5 NULL 9 Using index for group-by SELECT DISTINCT(a) FROM t1; a 1 @@ -2507,7 +2507,7 @@ a 4 EXPLAIN SELECT SQL_BIG_RESULT DISTINCT(a) FROM t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL idx 5 NULL 16 Using index +1 SIMPLE t1 range NULL idx 5 NULL 9 Using index for group-by SELECT SQL_BIG_RESULT DISTINCT(a) FROM t1; a 1 @@ -2646,7 +2646,7 @@ INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT a,b,c+1,d FROM t1; EXPLAIN SELECT DISTINCT c FROM t1 WHERE d=4; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL foo 20 NULL 32 Using where; Using index +1 SIMPLE t1 range NULL foo 10 NULL 9 Using where; Using index for group-by SELECT DISTINCT c FROM t1 WHERE d=4; c 1 @@ -3339,19 +3339,19 @@ INSERT INTO t2 SELECT a, b + 4, c,d,e,f FROM t2; INSERT INTO t2 SELECT a + 1, b, c,d,e,f FROM t2; EXPLAIN SELECT COUNT(DISTINCT a) FROM t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL a 10 NULL 16 Using index +1 SIMPLE t1 range NULL a 5 NULL 9 Using index for group-by SELECT COUNT(DISTINCT a) FROM t1; COUNT(DISTINCT a) 2 EXPLAIN SELECT COUNT(DISTINCT a,b) FROM t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL a 10 NULL 16 Using index +1 SIMPLE t1 range NULL a 10 NULL 9 Using index for group-by SELECT COUNT(DISTINCT a,b) FROM t1; COUNT(DISTINCT a,b) 16 EXPLAIN SELECT COUNT(DISTINCT b,a) FROM t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL a 10 NULL 16 Using index +1 SIMPLE t1 range NULL a 10 NULL 9 Using index for group-by SELECT COUNT(DISTINCT b,a) FROM t1; COUNT(DISTINCT b,a) 16 @@ -3414,7 +3414,7 @@ COUNT(DISTINCT a) 2 EXPLAIN SELECT 1 FROM t1 HAVING COUNT(DISTINCT a) < 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL a 10 NULL 16 Using index +1 SIMPLE t1 range NULL a 5 NULL 9 Using index for group-by SELECT 1 FROM t1 HAVING COUNT(DISTINCT a) < 10; 1 1 @@ -3435,19 +3435,19 @@ COUNT(DISTINCT t1_1.a) 1 EXPLAIN SELECT COUNT(DISTINCT a), 12 FROM t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL a 10 NULL 16 Using index +1 SIMPLE t1 range NULL a 5 NULL 9 Using index for group-by SELECT COUNT(DISTINCT a), 12 FROM t1; COUNT(DISTINCT a) 12 2 12 EXPLAIN SELECT COUNT(DISTINCT a, b, c) FROM t2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 index NULL a 15 NULL 16 Using index +1 SIMPLE t2 range NULL a 15 NULL 9 Using index for group-by SELECT COUNT(DISTINCT a, b, c) FROM t2; COUNT(DISTINCT a, b, c) 16 EXPLAIN SELECT COUNT(DISTINCT a), SUM(DISTINCT a), AVG(DISTINCT a) FROM t2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 index NULL a 15 NULL 16 Using index +1 SIMPLE t2 range NULL a 5 NULL 9 Using index for group-by SELECT COUNT(DISTINCT a), SUM(DISTINCT a), AVG(DISTINCT a) FROM t2; COUNT(DISTINCT a) SUM(DISTINCT a) AVG(DISTINCT a) 2 3 1.5000 @@ -3459,7 +3459,7 @@ COUNT(DISTINCT a) SUM(DISTINCT a) AVG(DISTINCT f) 2 3 1.0000 EXPLAIN SELECT COUNT(DISTINCT a, b), COUNT(DISTINCT b, a) FROM t2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 index NULL a 15 NULL 16 Using index +1 SIMPLE t2 range NULL a 10 NULL 9 Using index for group-by SELECT COUNT(DISTINCT a, b), COUNT(DISTINCT b, a) FROM t2; COUNT(DISTINCT a, b) COUNT(DISTINCT b, a) 16 16 @@ -3893,5 +3893,22 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index a a 13 NULL 2 Using where; Using index drop table t1; # +# MDEV-15433: Optimizer does not use group by optimization with distinct +# +CREATE TABLE t1 (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, a INT NOT NULL, KEY(a)); +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +EXPLAIN SELECT DISTINCT a FROM t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range NULL a 4 NULL 5 Using index for group-by +SELECT DISTINCT a FROM t1; +a +1 +2 +3 +4 +drop table t1; +# # End of 10.1 tests # diff --git a/mysql-test/suite/vcol/r/vcol_select_innodb.result b/mysql-test/suite/vcol/r/vcol_select_innodb.result index 5c29f5f8283..97bfbbe4eaf 100644 --- a/mysql-test/suite/vcol/r/vcol_select_innodb.result +++ b/mysql-test/suite/vcol/r/vcol_select_innodb.result @@ -133,7 +133,7 @@ count(distinct c) 3 explain select count(distinct c) from t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL c 5 NULL 5 Using index +1 SIMPLE t1 range NULL c 5 NULL 6 Using index for group-by (scanning) ### ### filesort & range-based utils ### diff --git a/mysql-test/suite/vcol/r/vcol_select_myisam.result b/mysql-test/suite/vcol/r/vcol_select_myisam.result index 7c371a1008a..6dee132b3e5 100644 --- a/mysql-test/suite/vcol/r/vcol_select_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_select_myisam.result @@ -133,7 +133,7 @@ count(distinct c) 3 explain select count(distinct c) from t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL c 5 NULL 5 Using index +1 SIMPLE t1 range NULL c 5 NULL 6 Using index for group-by (scanning) ### ### filesort & range-based utils ### diff --git a/mysql-test/t/group_min_max.test b/mysql-test/t/group_min_max.test index b0bc42d7f8c..adad9073235 100644 --- a/mysql-test/t/group_min_max.test +++ b/mysql-test/t/group_min_max.test @@ -1608,6 +1608,33 @@ explain select min(a) from t1 where a between "a" and "Cafeeeeeeeeeeeeeeeeeeeeee explain select min(a) from t1 where a between "abbbbbbbbbbbbbbbbbbbb" and "Cafe2"; drop table t1; +--echo # +--echo # MDEV-15433: Optimizer does not use group by optimization with distinct +--echo # + +CREATE TABLE t1 (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, a INT NOT NULL, KEY(a)); +--disable_query_log +INSERT INTO t1(a) VALUES (1), (2), (3), (4); +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +INSERT INTO t1(a) SELECT a FROM t1; +--enable_query_log +OPTIMIZE TABLE t1; +EXPLAIN SELECT DISTINCT a FROM t1; +SELECT DISTINCT a FROM t1; +drop table t1; + --echo # --echo # End of 10.1 tests --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 64d1565dec7..d1b1c17f178 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1388,6 +1388,14 @@ JOIN::optimize_inner() error= 1; DBUG_RETURN(1); } + if (!group_list) + { + /* The output has only one row */ + order=0; + simple_order=1; + group_optimized_away= 1; + select_distinct=0; + } } /* Calculate how to do the join */ @@ -5754,7 +5762,7 @@ add_group_and_distinct_keys(JOIN *join, JOIN_TAB *join_tab) Item_field *cur_item; key_map possible_keys(0); - if (join->group_list || join->simple_group) + if (join->group_list) { /* Collect all query fields referenced in the GROUP clause. */ for (cur_group= join->group_list; cur_group; cur_group= cur_group->next) (*cur_group->item)->walk(&Item::collect_item_field_processor, 0, From 998b1c0e75a9512bfec308da19096cbced4150ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sun, 5 Aug 2018 12:11:31 +0300 Subject: [PATCH 011/111] Fix galera test MW-44 --- mysql-test/suite/galera/r/MW-44.result | 15 ++------------- mysql-test/suite/galera/t/MW-44.test | 11 ++++++----- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/mysql-test/suite/galera/r/MW-44.result b/mysql-test/suite/galera/r/MW-44.result index 459a61030a4..a1e55318422 100644 --- a/mysql-test/suite/galera/r/MW-44.result +++ b/mysql-test/suite/galera/r/MW-44.result @@ -1,21 +1,10 @@ SET GLOBAL general_log='OFF'; TRUNCATE TABLE mysql.general_log; -SELECT COUNT(*) from mysql.general_log; -COUNT(*) -0 -SELECT * FROM mysql.general_log; -event_time user_host thread_id server_id command_type argument SET GLOBAL general_log='OFF'; TRUNCATE TABLE mysql.general_log; -SELECT COUNT(*) from mysql.general_log; -COUNT(*) -0 -SELECT * FROM mysql.general_log; -event_time user_host thread_id server_id command_type argument SET GLOBAL general_log='ON'; -SELECT COUNT(*) from mysql.general_log; -COUNT(*) -1 +SELECT argument from mysql.general_log WHERE argument NOT LIKE 'SELECT%'; +argument SET SESSION wsrep_osu_method=TOI; CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; SET SESSION wsrep_osu_method=RSU; diff --git a/mysql-test/suite/galera/t/MW-44.test b/mysql-test/suite/galera/t/MW-44.test index 09f444fdff4..6b5eb823985 100644 --- a/mysql-test/suite/galera/t/MW-44.test +++ b/mysql-test/suite/galera/t/MW-44.test @@ -8,18 +8,19 @@ --connection node_1 SET GLOBAL general_log='OFF'; TRUNCATE TABLE mysql.general_log; -SELECT COUNT(*) from mysql.general_log; -SELECT * FROM mysql.general_log; +--let $wait_condition = SELECT COUNT(*) = 0 FROM mysql.general_log; +--source include/wait_condition.inc --connection node_2 SET GLOBAL general_log='OFF'; TRUNCATE TABLE mysql.general_log; -SELECT COUNT(*) from mysql.general_log; -SELECT * FROM mysql.general_log; +--let $wait_condition = SELECT COUNT(*) = 0 FROM mysql.general_log; +--source include/wait_condition.inc --connection node_1 SET GLOBAL general_log='ON'; -SELECT COUNT(*) from mysql.general_log; +SELECT argument from mysql.general_log WHERE argument NOT LIKE 'SELECT%'; + SET SESSION wsrep_osu_method=TOI; CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; SET SESSION wsrep_osu_method=RSU; From 3b37edee1a5121e9523fa8a7f483185f402905e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 6 Aug 2018 15:45:44 +0300 Subject: [PATCH 012/111] MDEV-13333: Deadlock failure that does not occur elsewhere InnoDB executed code that is mean to execute only when Galera is used and in bad luck one of the transactions is selected incorrectly as deadlock victim. Fixed by adding wsrep_on_trx() condition before entering actual Galera transaction handling. No always repeatable test case for this issue is known. --- storage/innobase/lock/lock0lock.cc | 5 +++-- storage/xtradb/lock/lock0lock.cc | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 441fcace7c3..3970a559a4c 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4963,7 +4963,7 @@ lock_table_create( UT_LIST_ADD_LAST(trx_locks, trx->lock.trx_locks, lock); #ifdef WITH_WSREP - if (c_lock) { + if (c_lock && wsrep_on_trx(trx)) { if (wsrep_thd_is_wsrep(trx->mysql_thd) && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { UT_LIST_INSERT_AFTER( @@ -5202,9 +5202,10 @@ lock_table_enqueue_waiting( /* Enqueue the lock request that will wait to be granted */ #ifdef WITH_WSREP - if (trx->lock.was_chosen_as_deadlock_victim) { + if (trx->lock.was_chosen_as_deadlock_victim && wsrep_on_trx(trx)) { return(DB_DEADLOCK); } + lock = lock_table_create(c_lock, table, mode | LOCK_WAIT, trx); #else lock = lock_table_create(table, mode | LOCK_WAIT, trx); diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index 549cc411f69..2183d281b78 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -5003,7 +5003,7 @@ lock_table_create( UT_LIST_ADD_LAST(trx_locks, trx->lock.trx_locks, lock); #ifdef WITH_WSREP - if (c_lock) { + if (c_lock && wsrep_on_trx(trx)) { if (wsrep_thd_is_wsrep(trx->mysql_thd) && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { UT_LIST_INSERT_AFTER( @@ -5244,9 +5244,10 @@ lock_table_enqueue_waiting( /* Enqueue the lock request that will wait to be granted */ #ifdef WITH_WSREP - if (trx->lock.was_chosen_as_deadlock_victim) { + if (trx->lock.was_chosen_as_deadlock_victim && wsrep_on_trx(trx)) { return(DB_DEADLOCK); } + lock = lock_table_create(c_lock, table, mode | LOCK_WAIT, trx); #else lock = lock_table_create(table, mode | LOCK_WAIT, trx); From ebaacf0747bad9f41686a23f0b8ba4ff9f364f47 Mon Sep 17 00:00:00 2001 From: Rasmus Johansson Date: Mon, 6 Aug 2018 16:46:19 +0300 Subject: [PATCH 013/111] Update rules --- debian/dist/Ubuntu/rules | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/dist/Ubuntu/rules b/debian/dist/Ubuntu/rules index 5f3ea672770..e570a99c80a 100755 --- a/debian/dist/Ubuntu/rules +++ b/debian/dist/Ubuntu/rules @@ -1,7 +1,10 @@ #!/usr/bin/make -f export DH_VERBOSE=1 -export DEB_BUILD_HARDENING=1 +#export DEB_BUILD_HARDENING=1 +export DEB_BUILD_MAINT_OPTIONS = hardening=+all,-pie +DPKG_EXPORT_BUILDFLAGS = 1 +include /usr/share/dpkg/buildflags.mk PACKAGE=mariadb-5.5 From ad577091edf288e549c730933c514852b471991c Mon Sep 17 00:00:00 2001 From: Sachin Date: Mon, 6 Aug 2018 21:22:17 +0530 Subject: [PATCH 014/111] MDEV-16904 inline void swap(base_list &rhs) should swap list only when list is... not empty We should swap the list only when list is not empty. --- sql/sql_list.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/sql_list.h b/sql/sql_list.h index 08667bed02a..8956a786715 100644 --- a/sql/sql_list.h +++ b/sql/sql_list.h @@ -306,10 +306,13 @@ public: */ inline void swap(base_list &rhs) { + list_node **rhs_last=rhs.last; swap_variables(list_node *, first, rhs.first); - swap_variables(list_node **, last, rhs.last); swap_variables(uint, elements, rhs.elements); + rhs.last= last == &first ? &rhs.first : last; + last = rhs_last == &rhs.first ? &first : rhs_last; } + inline list_node* last_node() { return *last; } inline list_node* first_node() { return first;} inline void *head() { return first->info; } From 757e3b88d26f8282df83f42ea3c451d2d2dac3cf Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 7 Aug 2018 10:43:08 -0400 Subject: [PATCH 015/111] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c169837c460..f49c0084775 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=1 -MYSQL_VERSION_PATCH=35 +MYSQL_VERSION_PATCH=36 From 7e8ed15b95b64af2068a3515fe97f2c245311323 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 9 Aug 2018 02:24:12 +0000 Subject: [PATCH 016/111] Fixes after review and correction of the problems caused by the fact that during the SST innodb plugin is not yet initialized, as well as problems with running tests from the root user (not directly related to the MDEV-10754). --- .../galera/include/galera_wsrep_recover.inc | 9 ++++ .../galera/t/galera_sst_rsync_data_dir.cnf | 2 +- .../galera/t/galera_sst_rsync_data_dir.test | 3 +- scripts/wsrep_sst_common.sh | 2 + scripts/wsrep_sst_rsync.sh | 33 ++++++++++-- sql/wsrep_mysqld.h | 1 + sql/wsrep_sst.cc | 50 +++++++++++++++++++ storage/innobase/handler/ha_innodb.cc | 6 +++ 8 files changed, 99 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/galera/include/galera_wsrep_recover.inc b/mysql-test/suite/galera/include/galera_wsrep_recover.inc index 090ffe5f5df..7f1ed1a875d 100644 --- a/mysql-test/suite/galera/include/galera_wsrep_recover.inc +++ b/mysql-test/suite/galera/include/galera_wsrep_recover.inc @@ -1,5 +1,14 @@ --echo Performing --wsrep-recover ... +# +# When mysqld is run by a root user it will fail to start unless +# we specify what user is ran it (using "--user root" option): +# +if ($MYSQL_TEST_ROOT == 1) { +--exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --innodb --wsrep-recover --user root > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 +} +if ($MYSQL_TEST_ROOT != 1) { --exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --innodb --wsrep-recover > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 +} --perl use strict; diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf index d084740c8a6..afe9796a11a 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf +++ b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf @@ -2,10 +2,10 @@ [mysqld] wsrep_sst_method=rsync -innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test [mysqld.1] wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' [mysqld.2] +innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test_2 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test index 6220836c11d..68aa1068f75 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test @@ -13,5 +13,4 @@ --source include/auto_increment_offset_restore.inc # cleanup temporary database files: ---remove_files_wildcard $MYSQL_TMP_DIR/rsync_test * ---rmdir $MYSQL_TMP_DIR/rsync_test +--remove_files_wildcard $MYSQL_TMP_DIR/rsync_test_2 * diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index e160bf5c776..7f1f44d0cdb 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -27,6 +27,7 @@ WSREP_SST_OPT_PSWD=${WSREP_SST_OPT_PSWD:-} WSREP_SST_OPT_DEFAULT="" WSREP_SST_OPT_EXTRA_DEFAULT="" WSREP_SST_OPT_SUFFIX_DEFAULT="" +WSREP_SST_OPT_SUFFIX_VALUE="" while [ $# -gt 0 ]; do case "$1" in @@ -76,6 +77,7 @@ case "$1" in ;; '--defaults-group-suffix') readonly WSREP_SST_OPT_SUFFIX_DEFAULT="$1=$2" + readonly WSREP_SST_OPT_SUFFIX_VALUE="$2" shift ;; '--host') diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 6d8d5d1168a..0962e59f437 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -123,8 +123,16 @@ fi INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""} # if INNODB_DATA_HOME_DIR env. variable is not set, try to get it from my.cnf -if [ -z "INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf --mysqld innodb-data-home-dir '') +if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-data-home-dir '') +fi + +if [ -n "$INNODB_DATA_HOME_DIR" ]; then + # handle both relative and absolute paths + INNODB_DATA_HOME_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$INNODB_DATA_HOME_DIR"; cd $INNODB_DATA_HOME_DIR; pwd -P) +else + # default to datadir + INNODB_DATA_HOME_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P) fi # Old filter - include everything except selected @@ -135,8 +143,8 @@ fi # New filter - exclude everything except dirs (schemas) and innodb files FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' - -f '+ /wsrep_sst_binlog.tar' -f '+ $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '+ $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*') + -f '+ /wsrep_sst_binlog.tar' -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' + -f '- $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*'" SSTKEY=$(parse_cnf sst tkey "") SSTCERT=$(parse_cnf sst tcert "") @@ -242,6 +250,19 @@ EOF exit $RC fi + # Transfer InnoDB data files + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ + -f '- **' "$INNODB_DATA_HOME_DIR/" \ + rsync://$WSREP_SST_OPT_ADDR-data_dir >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_data_home_dir returned code $RC:" + exit 255 # unknown error + fi + # second, we transfer InnoDB log files rsync ${STUNNEL:+--rsh="$STUNNEL"} \ --owner --group --perms --links --specials \ @@ -340,8 +361,12 @@ $SILENT path = $WSREP_SST_OPT_DATA [$MODULE-log_dir] path = $WSREP_LOG_DIR +[$MODULE-data_dir] + path = $INNODB_DATA_HOME_DIR EOF +[ "$(whoami)" != root ] || sed -i '/read only = no/s/.*/&\nuid = root\ngid = root\n/' "$RSYNC_CONF" + # rm -rf "$DATA"/ib_logfile* # we don't want old logs around # listen at all interfaces (for firewalled setups) diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 272e4ac4984..9fc8aa02df4 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -160,6 +160,7 @@ extern "C" time_t wsrep_thd_query_start(THD *thd); extern "C" query_id_t wsrep_thd_query_id(THD *thd); extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd); extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id); +extern "C" void wsrep_set_data_home_dir(const char *data_dir); extern void wsrep_close_client_connections(my_bool wait_to_end); extern int wsrep_wait_committing_connections_close(int wait_time); diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 155051bfe84..0274a364bc5 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -67,6 +67,13 @@ bool wsrep_sst_method_update (sys_var *self, THD* thd, enum_var_type type) return 0; } +static const char* data_home_dir = NULL; + +extern "C" +void wsrep_set_data_home_dir(const char *data_dir) +{ + data_home_dir= (data_dir && *data_dir) ? data_dir : NULL; +} static void make_wsrep_defaults_file() { @@ -549,6 +556,29 @@ static int sst_append_auth_env(wsp::env& env, const char* sst_auth) return -env.error(); } +#define DATA_HOME_DIR_ENV "INNODB_DATA_HOME_DIR" + +static int sst_append_data_dir(wsp::env& env, const char* data_dir) +{ + int const data_dir_size= strlen(DATA_HOME_DIR_ENV) + 1 /* = */ + + (data_dir ? strlen(data_dir) : 0) + 1 /* \0 */; + + wsp::string data_dir_str(data_dir_size); // for automatic cleanup on return + if (!data_dir_str()) return -ENOMEM; + + int ret= snprintf(data_dir_str(), data_dir_size, "%s=%s", + DATA_HOME_DIR_ENV, data_dir ? data_dir : ""); + + if (ret < 0 || ret >= data_dir_size) + { + WSREP_ERROR("sst_append_data_dir(): snprintf() failed: %d", ret); + return (ret < 0 ? ret : -EMSGSIZE); + } + + env.append(data_dir_str()); + return -env.error(); +} + static ssize_t sst_prepare_other (const char* method, const char* sst_auth, const char* addr_in, @@ -610,6 +640,16 @@ static ssize_t sst_prepare_other (const char* method, return ret; } + if (data_home_dir) + { + if ((ret= sst_append_data_dir(env, data_home_dir))) + { + WSREP_ERROR("sst_prepare_other(): appending data " + "directory failed: %d", ret); + return ret; + } + } + pthread_t tmp; sst_thread_arg arg(cmd_str(), env()); mysql_mutex_lock (&arg.lock); @@ -1301,6 +1341,16 @@ wsrep_cb_status_t wsrep_sst_donate_cb (void* app_ctx, void* recv_ctx, return WSREP_CB_FAILURE; } + if (data_home_dir) + { + if ((ret= sst_append_data_dir(env, data_home_dir))) + { + WSREP_ERROR("wsrep_sst_donate_cb(): appending data " + "directory failed: %d", ret); + return WSREP_CB_FAILURE; + } + } + if (!strcmp (WSREP_SST_MYSQLDUMP, method)) { ret = sst_donate_mysqldump(data, ¤t_gtid->uuid, uuid_str, diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5667efea5df..d2701ecfc0b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3623,6 +3623,12 @@ innobase_init( srv_data_home = (innobase_data_home_dir ? innobase_data_home_dir : default_path); +#ifdef WITH_WSREP + /* If we use the wsrep API, then we need to tell the server + the path to the data files (for passing it to the SST scripts): */ + wsrep_set_data_home_dir(innobase_data_home_dir); +#endif /* WITH_WSREP */ + /* Set default InnoDB data file size to 12 MB and let it be auto-extending. Thus users can use InnoDB in >= 4.0 without having to specify any startup options. */ From 0496bbc120c9b583726b839aa8dedef5a5d4b647 Mon Sep 17 00:00:00 2001 From: faust Date: Thu, 9 Aug 2018 15:05:40 -0300 Subject: [PATCH 017/111] MDEV-15869 Mariabackup is lacking some dependencies declaration (#771) * Backport from 10.4 to resolve dependency problem Using the dependency syntax from 10.4 branch because {$LIBSSL} can not be used. --- debian/control | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 3048ec78d62..71329a870c9 100644 --- a/debian/control +++ b/debian/control @@ -449,5 +449,7 @@ Description: GSSAPI authentication plugin for MariaDB client Package: mariadb-backup-10.1 Section: database Architecture: any -Depends: libarchive12 | libarchive13 +Depends: mariadb-client-core-10.1 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends} Description: Backup tool for MariaDB server From 3ff0801c7397e3ae5fc538ffca3d58891cd4f27b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 11 Aug 2018 12:11:59 +0200 Subject: [PATCH 018/111] MDEV-16810 AddressSanitizer: stack-buffer-overflow in int10_to_str truncate incorrect values in convert_period_to_month() so that PERIOD_DIFF never returns a value outside of 2^23 range. And, for safety, increase buffer sizes for int10_to_str to be sufficienly big for any int10_to_str result. --- mysql-test/r/func_time.result | 3 +++ mysql-test/t/func_time.test | 4 ++++ sql/protocol.cc | 8 ++++---- sql/sql_time.cc | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index cffed8eae5b..3f0d99e1315 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -2657,6 +2657,9 @@ SEC_TO_TIME(MAKEDATE(0,RAND(~0))) 838:59:59 Warnings: Warning 1292 Truncated incorrect time value: '20000101' +SELECT PERIOD_DIFF(2018, AES_ENCRYPT('Rae Bareli', 'Rae Bareli')); +PERIOD_DIFF(2018, AES_ENCRYPT('Rae Bareli', 'Rae Bareli')) +24257 # # End of 5.5 tests # diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 8323bd30d2c..361eff170fb 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1629,6 +1629,10 @@ DO TO_DAYS(SEC_TO_TIME(MAKEDATE('',RAND(~(''))))); SELECT TO_DAYS(SEC_TO_TIME(MAKEDATE(0,RAND(~0)))); SELECT SEC_TO_TIME(MAKEDATE(0,RAND(~0))); +# +# MDEV-16810 AddressSanitizer: stack-buffer-overflow in int10_to_str +# +SELECT PERIOD_DIFF(2018, AES_ENCRYPT('Rae Bareli', 'Rae Bareli')); --echo # --echo # End of 5.5 tests diff --git a/sql/protocol.cc b/sql/protocol.cc index ac9fb1e9384..8602d9131c1 100644 --- a/sql/protocol.cc +++ b/sql/protocol.cc @@ -643,7 +643,7 @@ uchar *net_store_data(uchar *to, const uchar *from, size_t length) uchar *net_store_data(uchar *to,int32 from) { - char buff[20]; + char buff[22]; uint length=(uint) (int10_to_str(from,buff,10)-buff); to=net_store_length_fast(to,length); memcpy(to,buff,length); @@ -1060,7 +1060,7 @@ bool Protocol_text::store_tiny(longlong from) DBUG_ASSERT(field_types == 0 || field_types[field_pos] == MYSQL_TYPE_TINY); field_pos++; #endif - char buff[20]; + char buff[22]; return net_store_data((uchar*) buff, (size_t) (int10_to_str((int) from, buff, -10) - buff)); } @@ -1074,7 +1074,7 @@ bool Protocol_text::store_short(longlong from) field_types[field_pos] == MYSQL_TYPE_SHORT); field_pos++; #endif - char buff[20]; + char buff[22]; return net_store_data((uchar*) buff, (size_t) (int10_to_str((int) from, buff, -10) - buff)); @@ -1089,7 +1089,7 @@ bool Protocol_text::store_long(longlong from) field_types[field_pos] == MYSQL_TYPE_LONG); field_pos++; #endif - char buff[20]; + char buff[22]; return net_store_data((uchar*) buff, (size_t) (int10_to_str((long int)from, buff, (from <0)?-10:10)-buff)); diff --git a/sql/sql_time.cc b/sql/sql_time.cc index d912a7b78d6..33bb9a460b0 100644 --- a/sql/sql_time.cc +++ b/sql/sql_time.cc @@ -190,7 +190,7 @@ bool get_date_from_daynr(long daynr,uint *ret_year,uint *ret_month, ulong convert_period_to_month(ulong period) { ulong a,b; - if (period == 0) + if (period == 0 || period > 999912) return 0L; if ((a=period/100) < YY_PART_YEAR) a+=2000; From 5553d3f1f691a374124686946bd1f1cc4ef9fb45 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 12 Aug 2018 14:25:53 +0300 Subject: [PATCH 019/111] Custom set of tests which can be run with 10.0 clients --- mysql-test/collections/10.0-compatible.list | 591 ++++++++++++++++++++ 1 file changed, 591 insertions(+) create mode 100644 mysql-test/collections/10.0-compatible.list diff --git a/mysql-test/collections/10.0-compatible.list b/mysql-test/collections/10.0-compatible.list new file mode 100644 index 00000000000..415c74ab00c --- /dev/null +++ b/mysql-test/collections/10.0-compatible.list @@ -0,0 +1,591 @@ +main.1st +main.adddate_454 +main.almost_full +main.alter_table_autoinc-5574 +main.alter_table_errors +main.alter_table_mdev539_maria +main.alter_table_mdev539_myisam +main.alter_table_online +main.alter_table_trans +main.analyze +main.analyze_stmt +main.analyze_stmt_orderby +main.analyze_stmt_slow_query_log +main.ansi +main.assign_key_cache +main.auth_rpl +main.auto_increment +main.auto_increment_ranges_innodb +main.auto_increment_ranges_myisam +main.bad_frm_crash_5029 +main.bench_count_distinct +main.bigint +main.binary +main.binary_to_hex +main.blackhole +main.blackhole_plugin +main.bool +main.bootstrap +main.bug12427262 +main.bug13633383 +main.bug46760 +main.bug47671 +main.bulk_replace +main.case +main.change_user +main.check_constraint_show +main.client_xml +main.comment_column +main.comment_column2 +main.comment_index +main.comments +main.comment_table +main.commit_1innodb +main.compare +main.compound +main.contributors +main.count_distinct +main.count_distinct2 +main.create_drop_db +main.create_drop_event +main.create_drop_function +main.create_drop_index +main.create_drop_procedure +main.create_drop_server +main.create_drop_trigger +main.create_drop_user +main.create_drop_view +main.create_not_windows +main.create_select_tmp +main.create-uca +main.create_user +main.create_w_max_indexes_64 +main.ctype_ascii +main.ctype_big5 +main.ctype_binary +main.ctype_collate +main.ctype_cp1250_ch +main.ctype_cp1251 +main.ctype_cp850 +main.ctype_cp932 +main.ctype_cp932_binlog_row +main.ctype_cp932_binlog_stm +main.ctype_create +main.ctype_eucjpms +main.ctype_euckr +main.ctype_filename +main.ctype_filesystem +main.ctype_gb2312 +main.ctype_gbk +main.ctype_gbk_binlog +main.ctype_gbk_export_import +main.ctype_hebrew +main.ctype_latin1 +main.ctype_latin1_de +main.ctype_latin2 +main.ctype_latin2_ch +main.ctype_ldml +main.ctype_many +main.ctype_mb +main.ctype_nopad_8bit +main.ctype_partitions +main.ctype_recoding +main.ctype_sjis +main.ctype_swe7 +main.ctype_tis620 +main.ctype_uca +main.ctype_uca_innodb +main.ctype_uca_partitions +main.ctype_ucs +main.ctype_ucs2_def +main.ctype_ucs2_query_cache +main.ctype_ucs2_uca +main.ctype_ujis +main.ctype_ujis_ucs2 +main.ctype_upgrade +main.ctype_utf16 +main.ctype_utf16_def +main.ctype_utf16le +main.ctype_utf16_uca +main.ctype_utf32 +main.ctype_utf32_uca +main.ctype_utf8 +main.ctype_utf8mb4 +main.ctype_utf8mb4_heap +main.ctype_utf8mb4_innodb +main.ctype_utf8mb4_myisam +main.ctype_utf8mb4_uca +main.ctype_utf8_uca +main.date_formats +main.datetime_456 +main.default_storage_engine +main.delete +main.delete_returning +main.deprecated_features +main.derived_cond_pushdown +main.derived_opt +main.derived_view +main.distinct +main.drop-no_root +main.dyncol +main.empty_server_name-8224 +main.empty_table +main.endspace +main.enforce_storage_engine_opt +main.errors +main.events_2 +main.events_logs_tests +main.events_microsec +main.events_restart +main.events_scheduling +main.events_slowlog +main.events_trans +main.execution_constants +main.explain +main.explain_json +main.explain_json_format_partitions +main.explain_json_innodb +main.explain_non_select +main.ext_key_noPK_6794 +main.fast_prefix_index_fetch_innodb +main.features +main.filesort_bad_i_s-7585 +main.flush2 +main.foreign_key +main.frm_bad_row_type-7333 +main.fulltext +main.fulltext2 +main.fulltext3 +main.fulltext_cache +main.fulltext_charsets +main.fulltext_derived_4257 +main.fulltext_derived_4316 +main.fulltext_distinct +main.fulltext_left_join +main.fulltext_multi +main.fulltext_order_by +main.fulltext_update +main.fulltext_var +main.func_analyse +main.func_concat +main.func_crypt +main.func_date_add +main.func_default +main.func_des_encrypt +main.func_digest +main.func_encrypt +main.func_encrypt_ucs2 +main.func_equal +main.func_gconcat +main.func_group_innodb +main.func_hybrid_type +main.func_if +main.func_in +main.func_isnull +main.func_like +main.func_math +main.func_op +main.func_regexp +main.func_regexp_pcre +main.func_rollback +main.func_sapdb +main.func_set +main.func_system +main.func_test +main.func_time +main.func_time_hires +main.func_timestamp +main.function_defaults +main.function_defaults_innodb +main.gcc296 +main.get_diagnostics +main.gis +main.gis2 +main.gis-alter_table_online +main.gis-precise +main.gis-rt-precise +main.gis-rtree +main.grant_4332 +main.greedy_optimizer +main.group_by +main.group_by_innodb +main.group_by_null +main.group_min_max +main.group_min_max_innodb +main.handler_read_last +main.handlersocket +main.having +main.help +main.host_cache_size_functionality +main.huge_frm-6224 +main.implicit_char_to_num_conversion +main.implicit_commit +main.in_datetime_241 +main.index_intersect +main.index_intersect_innodb +main.index_merge_innodb +main.index_merge_myisam +main.information_schema2 +main.information_schema_all_engines +main.information_schema_chmod +main.information_schema_inno +main.information_schema_parameters +main.information_schema_part +main.information_schema_routines +main.information_schema_stats +main.init_file +main.init_file_longline_3816 +main.init_file_set_password-7656 +main.innodb_bug878769 +main.innodb_ext_key +main.innodb_group +main.innodb_icp +main.innodb_ignore_builtin +main.innodb_mrr_cpk +main.innodb_utf8 +main.insert +main.insert_innodb +main.insert_select +main.insert_update +main.insert_update_autoinc-7150 +main.join +main.join_cache +main.join_crash +main.join_nested +main.join_nested_jcl6 +main.join_optimizer +main.join_outer +main.join_outer_innodb +main.join_outer_jcl6 +main.key +main.key_cache +main.key_diff +main.key_primary +main.keyread +main.keywords +main.last_value +main.limit +main.limit_rows_examined +main.loaddata_autocom_innodb +main.locale +main.log_errchk +main.log_slow +main.log_state_bug33693 +main.log_tables_upgrade +main.long_tmpdir +main.lowercase_mixed_tmpdir +main.lowercase_table +main.lowercase_table5 +main.lowercase_table_grant +main.lowercase_table_qcache +main.lowercase_utf8 +main.lowercase_view +main.mdev13607 +main.mdev_14586 +main.mdev316 +main.mdl +main.merge_innodb +main.merge_mmap +main.metadata +main.mix2_myisam_ucs2 +main.mrr_derived_crash_4610 +main.mrr_icp_extra +main.multi_statement +main.multi_update2 +main.multi_update_innodb +main.multi_update_tiny_hash +main.myisam-blob +main.myisam_enable_keys-10506 +main.myisam_explain_non_select_all +main.myisam_icp +main.myisam_mrr +main.myisampack +main.myisam-system +main.mysql +main.mysql5613mysql +main.mysql57_virtual +main.mysqladmin +main.mysql_binary_mode +main.mysqlcheck +main.mysql_comments +main.mysql_cp932 +main.mysqld--defaults-file +main.mysqld--help +main.mysqld_help_crash-9183 +main.mysqld_option_err +main.mysqldump-compat +main.mysqldump-nl +main.mysqldump-no-binlog +main.mysqldump_restore +main.mysql_not_windows +main.mysql_protocols +main.mysqlshow +main.mysqlslap +main.mysqltest_256 +main.mysqltest_cont_on_error +main.mysql_tzinfo_to_sql_symlink +main.mysql_upgrade_noengine +main.mysql_upgrade_no_innodb +main.mysql_upgrade_ssl +main.mysql_upgrade_view +main.negation_elimination +main.no_binlog +main.no_password_column-mdev-11170 +main.null +main.null_key +main.odbc +main.olap +main.old-mode +main.order_by +main.order_by_innodb +main.order_by-mdev-10122 +main.order_by_optimizer +main.order_by_optimizer_innodb +main.order_by_sortkey +main.order_by_zerolength-4285 +main.order_fill_sortbuf +main.outfile_loaddata +main.parser +main.parser_bug21114_innodb +main.parser_precedence +main.parser_stack +main.partition +main.partition_binlog +main.partition_binlog_stmt +main.partition_blackhole +main.partition_bug18198 +main.partition_cache_innodb +main.partition_cache_myisam +main.partition_charset +main.partition_column +main.partition_column_prune +main.partition_datatype +main.partition_disabled +main.partition_error +main.partition_example +main.partition_exchange +main.partition_explicit_prune +main.partition_hash +main.partition_key_cache +main.partition_list +main.partition_mgm +main.partition_mgm_err +main.partition_mgm_err2 +main.partition_myisam +main.partition_not_blackhole +main.partition_not_windows +main.partition_order +main.partition_pruning +main.partition_range +main.partition_rename_longfilename +main.partition_truncate +main.partition_utf8 +main.perror +main.plugin +main.plugin_auth_qa +main.plugin_auth_qa_2 +main.plugin_auth_qa_3 +main.plugin_innodb +main.plugin_load +main.plugin_loaderr +main.plugin_load_option +main.plugin_maturity +main.preload +main.profiling +main.progress_976225 +main.ps_10nestset +main.ps_11bugs +main.ps_1general +main.ps_2myisam +main.ps_3innodb +main.ps_4heap +main.ps_5merge +main.ps_change_master +main.ps_ddl1 +main.ps_max_subselect-5113 +main.ps_not_windows +main.query_cache +main.query_cache_innodb +main.query_cache_merge +main.query_cache_with_views +main.range +main.range_innodb +main.range_mrr_icp +main.range_vs_index_merge +main.range_vs_index_merge_innodb +main.renamedb +main.reopen_temp_table +main.repair +main.repair_symlink-5543 +main.replace +main.rollback +main.round +main.row +main.row-checksum +main.row-checksum-old +main.rowid_order_innodb +main.rpl_mysqldump_slave +main.second_frac-9175 +main.select +main.select_found +main.selectivity +main.selectivity_innodb +main.selectivity_no_engine +main.select_jcl6 +main.select_pkeycache +main.select_safe +main.servers +main.set_password +main.set_statement_notembedded +main.set_statement_notembedded_binlog +main.show +main.show_bad_definer-5553 +main.show_create_user +main.show_function_with_pad_char_to_full_length +main.show_profile +main.show_row_order-9226 +main.sighup-6580 +main.signal +main.signal_demo1 +main.signal_demo2 +main.signal_demo3 +main.signal_sqlmode +main.single_delete_update +main.single_delete_update_innodb +main.skip_grants +main.skip_log_bin +main.sp-big +main.sp-bugs +main.sp-bugs2 +main.sp-destruct +main.sp-dynamic +main.sp-error +main.sp-fib +main.sp_gis +main.sp-group +main.sp_missing_4665 +main.sp-no-code +main.sp-prelocking +main.sp_stress_case +main.sp_trans +main.sp_trans_log +main.sp-ucs2 +main.sp-vars +main.ssl_7937 +main.ssl_8k_key +main.ssl_and_innodb +main.ssl_ca +main.ssl_cert_verify +main.ssl_connect +main.ssl_crl_clients +main.stack-crash +main.statistics +main.statistics_index_crash-7362 +main.stat_tables +main.stat_tables_disabled +main.stat_tables_innodb +main.stat_tables_partition +main.stat_tables_repl +main.strict +main.strict_autoinc_1myisam +main.strict_autoinc_2innodb +main.strict_autoinc_3heap +main.str_to_datetime_457 +main.subselect2 +main.subselect3 +main.subselect3_jcl6 +main.subselect4 +main.subselect_cache +main.subselect-crash_15755 +main.subselect_exists2in +main.subselect_exists2in_costmat +main.subselect_extra +main.subselect_extra_no_semijoin +main.subselect_gis +main.subselect_innodb +main.subselect_mat +main.subselect_mat_cost +main.subselect_mat_cost_bugs +main.subselect_notembedded +main.subselect_nulls +main.subselect_partial_match +main.subselect_sj2 +main.subselect_sj2_jcl6 +main.subselect_sj2_mat +main.subselect_sj_aria +main.subselect_sj_mat +main.subselect_sj_nonmerged +main.sum_distinct +main.sysdate_is_now +main.system_mysql_db +main.system_mysql_db_refs +main.table_elim +main.table_elim_debug +main.table_keyinfo-6838 +main.tablelock +main.table_options +main.table_options-5867 +main.temporal_literal +main.temporal_scale_4283 +main.temp_table_frm +main.timezone +main.timezone2 +main.timezone3 +main.timezone4 +main.tmp_table_count-7586 +main.trigger_no_defaults-11698 +main.trigger_null-8605 +main.truncate +main.truncate_badse +main.truncate-stale-6500 +main.type_binary +main.type_bit +main.type_bit_innodb +main.type_blob +main.type_datetime_hires +main.type_decimal +main.type_enum +main.type_float +main.type_int +main.type_nchar +main.type_newdecimal +main.type_num +main.type_num_innodb +main.type_ranges +main.type_set +main.type_temporal_innodb +main.type_temporal_mysql56 +main.type_time +main.type_time_6065 +main.type_time_hires +main.type_timestamp_hires +main.type_uint +main.type_varchar +main.type_year +main.update_ignore_216 +main.update_innodb +main.upgrade +main.user_var +main.varbinary +main.variables_community +main.view_alias +main.warnings_engine_disabled +main.win_avg +main.win_big +main.win_big-mdev-10092 +main.win_big-mdev-11697 +main.win_bit +main.win_empty_over +main.win_first_last_value +main.win_insert_select +main.win_i_s +main.win_lead_lag +main.win_min_max +main.win_nth_value +main.win_orderby +main.win_percent_cume +main.win_rank +main.win_sum +main.xa_binlog +main.xml +main.xtradb_mrr From 36832711c15cc3208f4d76ff6677488ea4bb11a4 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 13 Aug 2018 10:40:47 +0200 Subject: [PATCH 020/111] Reverting changes made to support the mtr under the root --- mysql-test/suite/galera/include/galera_wsrep_recover.inc | 9 --------- scripts/wsrep_sst_rsync.sh | 2 -- 2 files changed, 11 deletions(-) diff --git a/mysql-test/suite/galera/include/galera_wsrep_recover.inc b/mysql-test/suite/galera/include/galera_wsrep_recover.inc index 7f1ed1a875d..090ffe5f5df 100644 --- a/mysql-test/suite/galera/include/galera_wsrep_recover.inc +++ b/mysql-test/suite/galera/include/galera_wsrep_recover.inc @@ -1,14 +1,5 @@ --echo Performing --wsrep-recover ... -# -# When mysqld is run by a root user it will fail to start unless -# we specify what user is ran it (using "--user root" option): -# -if ($MYSQL_TEST_ROOT == 1) { ---exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --innodb --wsrep-recover --user root > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 -} -if ($MYSQL_TEST_ROOT != 1) { --exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --innodb --wsrep-recover > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 -} --perl use strict; diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 0962e59f437..f787262085b 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -365,8 +365,6 @@ $SILENT path = $INNODB_DATA_HOME_DIR EOF -[ "$(whoami)" != root ] || sed -i '/read only = no/s/.*/&\nuid = root\ngid = root\n/' "$RSYNC_CONF" - # rm -rf "$DATA"/ib_logfile* # we don't want old logs around # listen at all interfaces (for firewalled setups) From 074b672b5d94d291afce5f6541f39d68c65caa62 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 13 Aug 2018 19:43:59 +0100 Subject: [PATCH 021/111] MDEV-16963 Tighten named pipe access control Use real DACL instead of NULL DACL. Grant Everyone just read/write access to pipe (instead of all access like previously with NULL ACL) --- sql/mysqld.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5a9aba7f2e4..aa749e5aaef 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1195,9 +1195,9 @@ static NTService Service; ///< Service object for WinNT #endif /* __WIN__ */ #ifdef _WIN32 +#include /* ConvertStringSecurityDescriptorToSecurityDescriptor */ static char pipe_name[512]; static SECURITY_ATTRIBUTES saPipeSecurity; -static SECURITY_DESCRIPTOR sdPipeDescriptor; static HANDLE hPipe = INVALID_HANDLE_VALUE; #endif @@ -2238,21 +2238,20 @@ static void network_init(void) strxnmov(pipe_name, sizeof(pipe_name)-1, "\\\\.\\pipe\\", mysqld_unix_port, NullS); - bzero((char*) &saPipeSecurity, sizeof(saPipeSecurity)); - bzero((char*) &sdPipeDescriptor, sizeof(sdPipeDescriptor)); - if (!InitializeSecurityDescriptor(&sdPipeDescriptor, - SECURITY_DESCRIPTOR_REVISION)) + /* + Create a security descriptor for pipe. + - Use low integrity level, so that it is possible to connect + from any process. + - Give Everyone read/write access to pipe. + */ + if (!ConvertStringSecurityDescriptorToSecurityDescriptor( + "S:(ML;; NW;;; LW) D:(A;; FRFW;;; WD)", + SDDL_REVISION_1, &saPipeSecurity.lpSecurityDescriptor, NULL)) { sql_perror("Can't start server : Initialize security descriptor"); unireg_abort(1); } - if (!SetSecurityDescriptorDacl(&sdPipeDescriptor, TRUE, NULL, FALSE)) - { - sql_perror("Can't start server : Set security descriptor"); - unireg_abort(1); - } saPipeSecurity.nLength = sizeof(SECURITY_ATTRIBUTES); - saPipeSecurity.lpSecurityDescriptor = &sdPipeDescriptor; saPipeSecurity.bInheritHandle = FALSE; if ((hPipe= CreateNamedPipe(pipe_name, PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE, @@ -5859,6 +5858,7 @@ pthread_handler_t handle_connections_namedpipes(void *arg) thd->security_ctx->host= my_strdup(my_localhost, MYF(0)); create_new_thread(thd); } + LocalFree(saPipeSecurity.lpSecurityDescriptor); CloseHandle(connectOverlapped.hEvent); DBUG_LEAVE; decrement_handler_count(); From ca26f91bcaa21933147974c823852a2e1c2e2bd7 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Tue, 14 Aug 2018 03:35:06 +0300 Subject: [PATCH 022/111] Updated list of unstable tests for 10.3.9 release --- mysql-test/unstable-tests | 277 +++++++++++++++++++++++--------------- 1 file changed, 168 insertions(+), 109 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index d97792355b4..2c28c9a663b 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,7 +23,7 @@ # ############################################################################## -# Based on bb-10.3-release 36e59752e7fc70bc5179a3d730b5ce3ee58e4e30 +# Based on bb-10.3-release fac3e575b203e8d6a522a475f9aab4ec5041b146 #----------------------------------------------------------------------- @@ -32,12 +32,17 @@ archive.archive_symlink : MDEV-12170 archive.discover : MDEV-10510 - Table is marked as crashed archive.mysqlhotcopy_archive : MDEV-10995 - Hang on debug +archive-test_sql_discovery.discover : MDEV-16817 - Table marked as crashed + #----------------------------------------------------------------------- binlog.binlog_commit_wait : MDEV-10150 - Mismatch binlog.binlog_flush_binlogs_delete_domain : MDEV-14431 - Wrong exit code binlog.binlog_killed : MDEV-12925 - Wrong result +binlog.binlog_tmp_table_row : Added in 10.1.35, 10.2.17 binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint +binlog.load_data_stm_view : MDEV-16948 - Wrong result +binlog_tmp_table_row : Added in 10.3.9 #----------------------------------------------------------------------- @@ -47,6 +52,8 @@ binlog_encryption.encrypted_master_switch_to_unencrypted : MDEV-14190 binlog_encryption.encrypted_slave : Modified in 10.3.8 binlog_encryption.encryption_combo : MDEV-14199 - Table is marked as crashed binlog_encryption.rpl_binlog_errors : MDEV-12742 - Crash +binlog_encryption.rpl_checksum : MDEV-16951 - Wrong result +binlog_encryption.rpl_gtid_basic : MDEV-16947 - Server failed to start binlog_encryption.rpl_loadfile : MDEV-16645 - Timeout in include binlog_encryption.rpl_parallel : MDEV-10653 - Timeout in include binlog_encryption.rpl_relayrotate : MDEV-15194 - Timeout @@ -55,17 +62,13 @@ binlog_encryption.rpl_skip_replication : MDEV-13571 binlog_encryption.rpl_ssl : MDEV-14507 - Timeouts binlog_encryption.rpl_stm_relay_ign_space : MDEV-13278 - Wrong result (test assertion) binlog_encryption.rpl_sync : MDEV-13830 - Assertion failure +binlog_encryption.rpl_typeconv : MDEV-14362 - Lost connection to MySQL server during query #----------------------------------------------------------------------- -compat/oracle.column_compression : Added in 10.3.7 -compat/oracle.func_concat : Modified in 10.3.7 -compat/oracle.gis : Added in 10.3.7 +compat/oracle.events : Added in 10.3.9 compat/oracle.parser : Modified in 10.3.8 compat/oracle.sp-cursor-rowtype : Modified in 10.3.8 -compat/oracle.table_value_constr : Added in 10.3.7 -compat/oracle.versioning : Added in 10.3.7 -compat/oracle.win : Added in 10.3.7 #----------------------------------------------------------------------- @@ -79,7 +82,7 @@ encryption.create_or_replace : MDEV-12694 encryption.debug_key_management : MDEV-13841 - Timeout encryption.encrypt_and_grep : MDEV-13765 - Wrong result encryption.innochecksum : MDEV-13644 - Assertion failure -encryption.innodb-checksum-algorithm : MDEV-12898 - Deadlock of threads +encryption.innodb-checksum-algorithm : MDEV-12898 - Deadlock of threads; MDEV-16896 - Server crash encryption.innodb-compressed-blob : MDEV-14728 - Unable to get certificate encryption.innodb-discard-import : Modified in 10.3.8 encryption.innodb-encryption-alter : MDEV-13566 - Lock wait timeout @@ -97,6 +100,7 @@ encryption.innodb_encryption_discard_import : MDEV-16116 encryption.innodb_encryption_filekeys : MDEV-15673 - Timeout encryption.innodb_encryption_row_compressed : MDEV-16113 - Crash encryption.innodb_first_page : MDEV-10689 - Crash +encryption.innodb_lotoftables : MDEV-16111 - Wrong result encryption.innodb_scrub : MDEV-8139 - scrubbing tests need fixing encryption.innodb_scrub_background : MDEV-8139 - scrubbing tests need fixing encryption.innodb_scrub_compressed : MDEV-8139 - scrubbing tests need fixing @@ -109,18 +113,17 @@ engines/rr_trx.* : MDEV-10998 #----------------------------------------------------------------------- -federated.assisted_discovery : Modified in 10.3.7 +federated.assisted_discovery : Include file modified in 10.0.36 federated.federated_bug_35333 : MDEV-13410 - Wrong result federated.federated_bug_585688 : MDEV-14805 - Server crash, MDEV-12907 - Valgrind federated.federated_innodb : MDEV-10617 - Wrong checksum federated.federated_partition : MDEV-10417 - Fails on Mips federated.federated_transactions : MDEV-10617 - Wrong checksum federated.federatedx : MDEV-10617 - Wrong checksum -federated.federatedx_versioning : Added in 10.3.7 -federated.timestamps : Added in 10.3.7 #----------------------------------------------------------------------- +funcs_1.is_engines_federated : Include file modified in 10.0.36 funcs_1.memory_views : MDEV-11773 - timeout funcs_1.processlist_val_no_prot : MDEV-11223 - Wrong result funcs_1.processlist_val_ps : MDEV-12175 - Wrong plan @@ -153,63 +156,73 @@ galera_3nodes.* : Suite is no #----------------------------------------------------------------------- +gcol.gcol_rollback : MDEV-16954 - Unknown storage engine 'InnoDB' +gcol.gcol_update : Include file modified in 10.2.17, 10.3.9 +gcol.innodb_virtual_basic : MDEV-16950 - Failing assertion gcol.innodb_virtual_debug : MDEV-14134 - Crash, assertion failure; modified in 10.3.8 -gcol.innodb_virtual_index : Modified in 10.3.7 +gcol.innodb_virtual_debug_purge : MDEV-16952 - Wrong result; modified in 10.2.17, 10.3.9 +gcol.innodb_virtual_index : Include file modified in 10.2.17, 10.3.9 +gcol.innodb_virtual_purge : Include file modified in 10.3.9 #----------------------------------------------------------------------- -handler.heap : Modified in 10.3.7 -handler.innodb : Modified in 10.3.7 -handler.interface : Modified in 10.3.7 handler.ps : Added in 10.3.8 #----------------------------------------------------------------------- -heap.heap_auto_increment : MDEV-16652 - Out of range, modified in 10.3.8 +heap.heap_auto_increment : Modified in 10.3.8 #----------------------------------------------------------------------- innodb.101_compatibility : MDEV-13891 - Wrong result innodb.alter_copy : MDEV-16181 - Assertion failure -innodb.alter_foreign_crash : Added in 10.3.7 -innodb.alter_kill : MDEV-16273 - Unknown storage engine 'InnoDB'; added in 10.3.7 -innodb.alter_non_null : Added in 10.3.8 -innodb.alter_non_null_debug : Added in 10.3.8 -innodb.alter_not_null : Modified in 10.3.8 -innodb.alter_partitioned : Added in 10.3.7 -innodb.alter_rename_files : Added in 10.3.7 +innodb.alter_crash : MDEV-16944 - The process cannot access the file +innodb.alter_foreign_crash : Added in 10.2.16 +innodb.alter_kill : MDEV-16273 - Unknown storage engine 'InnoDB', MDEV-16946 - Wrong result +innodb.alter_not_null : Modified in 10.3.9 +innodb.alter_not_null_debug : Added in 10.3.9 +innodb.alter_partitioned_xa : Added in 10.0.36 +innodb.alter_rename_files : Added in 10.2.16 innodb.alter_sql_mode : Combinations added in 10.3.8 -innodb.analyze_table : Added in 10.3.7 +innodb.analyze_table : Added in 10.2.16 innodb.autoinc_persist : MDEV-15282 - Assertion failure innodb.binlog_consistent : MDEV-10618 - Server fails to start +innodb.dml_purge : Include file modified in 10.3.9 innodb.doublewrite : MDEV-12905 - Server crash innodb.group_commit_crash : MDEV-14191 - InnoDB registration failed innodb.group_commit_crash_no_optimize_thread : MDEV-13830 - Assertion failure +innodb.index_merge_threshold : Include files modified in 10.2.17, 10.3.9 +innodb.innodb-16k : Modified in 10.3.9 +innodb.innodb-32k-crash : MDEV-16953 - Corrupt log record found innodb.innodb-64k-crash : MDEV-13872 - Failure and crash on startup +innodb.innodb-alter : Modified in 10.0.36 innodb.innodb-alter-debug : MDEV-13182 - InnoDB: adjusting FSP_SPACE_FLAGS innodb.innodb-alter-table : MDEV-10619 - Testcase timeout innodb.innodb-alter-tempfile : MDEV-15285 - Table already exists innodb.innodb-alter-timestamp : Modified in 10.3.8 innodb.innodb-autoinc : Modified in 10.3.8 innodb.innodb-blob : Modified in 10.3.8 +innodb.innodb-corrupted-table : Modified in 10.3.9 innodb.innodb-fk : MDEV-13832 - Assertion failure on shutdown innodb.innodb-get-fk : MDEV-13276 - Server crash innodb.innodb-index-online : MDEV-14809 - Cannot save statistics +innodb.innodb-mdev-7513 : Modified in 10.3.9 innodb.innodb-mdev7046 : Modified in 10.3.8 -innodb.innodb-online-alter-gis : Modified in 10.3.7 +innodb.innodb-online-alter-gis : Modified in 10.2.16 innodb.innodb-page_compression_default : MDEV-13644 - Assertion failure; modified in 10.3.8 innodb.innodb-page_compression_lzma : MDEV-14353 - Wrong result innodb.innodb-page_compression_snappy : /MDEV-13644 - Assertion failure; modified in 10.3.8 innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb-table-online : MDEV-13894 - Wrong result; modified in 10.3.8 -innodb.innodb-wl5522 : MDEV-13644 - Assertion failure +innodb.innodb-wl5522 : MDEV-13644 - Assertion failure; modified in 10.3.9 innodb.innodb-wl5522-debug : MDEV-14200 - Wrong errno -innodb.innodb_bug13510739 : Modified in 10.3.7 innodb.innodb_bug14147491 : MDEV-11808 - Index is corrupt innodb.innodb_bug30423 : MDEV-7311 - Wrong result innodb.innodb_bug48024 : MDEV-14352 - Assertion failure -innodb.innodb_bug54044 : Modified in 10.3.7 +innodb.innodb_bug54044 : Modified in 10.2.16 innodb.innodb_bug59641 : MDEV-13830 - Assertion failure +innodb.innodb_buffer_pool_resize : MDEV-16964 - Assertion failure +innodb.innodb_buffer_pool_resize_with_chunks : MDEV-16964 - Assertion failure innodb.innodb_bulk_create_index_replication : MDEV-15273 - Slave failed to start innodb.innodb_defrag_stats_many_tables : MDEV-14198 - Table is full innodb.innodb_defragment_small : Modified in 10.3.8 @@ -219,49 +232,60 @@ innodb.innodb_max_recordsize_64k : MDEV-15203 innodb.innodb_monitor : MDEV-10939 - Testcase timeout innodb.innodb_query_cache : Added in 10.3.8 innodb.innodb_stats : MDEV-10682 - wrong result +innodb.innodb_stats_persistent : Include file modified in 10.2.17, 10.3.9 innodb.innodb_stats_persistent_debug : MDEV-14801 - Operation failed innodb.innodb_sys_semaphore_waits : MDEV-10331 - Semaphore wait innodb.innodb_zip_innochecksum2 : MDEV-13882 - Extra warnings innodb.innodb_zip_innochecksum3 : MDEV-14486 - Resource temporarily unavailable +innodb.instant_alter : Modified in 10.3.9 +innodb.instant_alter_crash : Include file modified in 10.3.9 +innodb.instant_alter_debug : Modified in 10.3.9 +innodb.instant_alter_rollback : Include file modified in 10.3.9 innodb.log_corruption : MDEV-13251 - Wrong result innodb.log_data_file_size : MDEV-14204 - Server failed to start innodb.log_file_name : MDEV-14193 - Exception innodb.log_file_size : MDEV-15668 - Not found pattern -innodb.monitor : MDEV-16179 - Wrong result -innodb.purge_secondary : MDEV-15681 - Wrong result +innodb.monitor : MDEV-16179 - Wrong result; modified in 10.3.9 +innodb.purge_secondary : MDEV-15681 - Wrong result; include file modified in 10.3.9 innodb.purge_thread_shutdown : MDEV-13792 - Wrong result innodb.read_only_recovery : MDEV-13886 - Server crash innodb.recovery_shutdown : MDEV-15671 - Checksum mismatch in datafile innodb.rename_table : Modified in 10.3.8 innodb.row_format_redundant : MDEV-15192 - Trying to access missing tablespace innodb.sp_temp_table : MDEV-16647 - Could not remove temporary table +innodb.strict_mode : Modified in 10.3.9 innodb.table_definition_cache_debug : MDEV-14206 - Extra warning innodb.table_flags : MDEV-13572 - Wrong result innodb.temp_table_savepoint : MDEV-16182 - Wrong result -innodb.temporary_table : MDEV-13265 - Wrong result; modified in 10.3.7 -innodb.tmpdir : Modified in 10.3.7 +innodb.temporary_table : MDEV-13265 - Wrong result; modified in 10.3.9 +innodb.truncate_purge_debug : Include file modified in 10.2.17, 10.3.9 innodb.update_time : MDEV-14804 - Wrong result +innodb.undo_log : Include file modified in 10.2.17, 10.3.9 innodb.xa_recovery : MDEV-15279 - mysqld got exception #----------------------------------------------------------------------- -innodb_fts.fulltext2 : Modified in 10.3.7 -innodb_fts.fulltext_var : Modified in 10.3.7 +innodb_fts.basic : Added in 10.0.36 +innodb_fts.fts_kill_query : Added in 10.1.35, 10.2.17, 10.3.9 innodb_fts.innodb-fts-fic : MDEV-14154 - Assertion failure -innodb_fts.innodb_fts_misc : Modified in 10.3.7 innodb_fts.innodb_fts_misc_debug : MDEV-14156 - Unexpected warning +innodb_fts.innodb_fts_multiple_index : Modified in 10.3.9 innodb_fts.innodb_fts_plugin : MDEV-13888 - Errors in server log innodb_fts.innodb_fts_stopword_charset : MDEV-13259 - Table crashed innodb_fts.sync : MDEV-14808 - Wrong result +innodb_fts.sync_ddl : Added in 10.1.35, 10.2.17, 10.3.9 #----------------------------------------------------------------------- +innodb_gis.create_spatial_index : Modified in 10.2.17, 10.3.9 +innodb_gis.kill_server : MDEV-16941 - Checksum mismatch +innodb_gis.rtree_compress : Include file modified in 10.2.17, 10.3.9 innodb_gis.rtree_compress2 : MDEV-16269 - Wrong result innodb_gis.rtree_concurrent_srch : MDEV-15284 - Wrong result with embedded -innodb_gis.rtree_purge : MDEV-15275 - Timeout +innodb_gis.rtree_purge : MDEV-15275 - Timeout; include file modified in 10.3.9 innodb_gis.rtree_recovery : MDEV-15274 - Error on check innodb_gis.rtree_split : MDEV-14208 - Too many arguments -innodb_gis.rtree_undo : MDEV-14456 - Timeout in include file +innodb_gis.rtree_undo : MDEV-14456 - Timeout in include file; include file modified in 10.3.9 innodb_gis.types : MDEV-15679 - Table is marked as crashed #----------------------------------------------------------------------- @@ -276,51 +300,70 @@ innodb_zip.wl6501_scale_1 : MDEV-13254 #----------------------------------------------------------------------- main.alter_table : Modified in 10.3.8 -main.alter_table_errors : Added in 10.3.7 main.alter_table_trans : MDEV-12084 - timeout main.analyze_stmt_slow_query_log : MDEV-12237 - Wrong result -main.ansi : Modified in 10.3.7 +main.assign_key_cache : Added in 10.0.36 +main.assign_key_cache_debug : Added in 10.0.36 main.auth_named_pipe : MDEV-14724 - System error 2 -main.auto_increment : MDEV-16652 - Out of range, modified in 10.3.8 +main.auto_increment : Modified in 10.3.8 +main.bootstrap : Modified in 10.1.35, 10.2.17, 10.3.9 +main.check_constraint : Modified in 10.2.17, 10.3.9 main.check : Modified in 10.3.8 -main.check_constraint : Modified in 10.3.7 -main.connect : MDEV-16270 - Wrong result +main.column_compression : Modified in 10.3.9 main.connect2 : MDEV-13885 - Server crash +main.connect_debug : Added in 10.0.36 +main.connect : MDEV-16270 - Wrong result main.count_distinct2 : MDEV-11768 - timeout main.create_delayed : MDEV-10605 - failed with timeout main.create_drop_event : MDEV-16271 - Wrong result -main.create_or_replace : Modified in 10.3.7 +main.create_or_replace : Modified in 10.2.16 +main.create_replace_tmp : Added in 10.2.17, 10.3.9 main.cte_nonrecursive : Modified in 10.3.8 -main.cte_recursive : Modified in 10.3.7 -main.ctype_utf16le : MDEV-10675: timeout or extra warnings -main.custom_aggregate_functions : Modified in 10.3.7 +main.cte_recursive : Modified in 10.2.17, 10.3.9 +main.cte_recursive_not_embedded : Added in 10.2.17, 10.3.9 +main.ctype_binary : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_eucjpms : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_euckr : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_gbk : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_latin1 : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_ucs : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_ujis : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_upgrade : MDEV-16945 - Error upon mysql_upgrade +main.ctype_utf16le : MDEV-10675: timeout or extra warnings; modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_utf16 : MDEV-10675: timeout or extra warnings; modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_utf32 : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_utf8mb4 : Modified in 10.1.35, 10.2.17, 10.3.9 +main.ctype_utf8 : Modified in 10.1.35, 10.2.17, 10.3.9 main.debug_sync : MDEV-10607 - internal error -main.derived_cond_pushdown : Modified in 10.3.8 +main.derived_cond_pushdown : Modified in 10.2.17, 10.3.9 +main.derived : Modified in 10.1.35, 10.2.17, 10.3.9 main.derived_opt : MDEV-11768 - timeout main.derived_view : Modified in 10.3.8 -main.distinct : MDEV-14194 - Crash; modified in 10.3.7 +main.distinct : MDEV-14194 - Crash main.drop_bad_db_type : MDEV-15676 - Wrong result main.events_2 : MDEV-13277 - Crash main.events_bugs : MDEV-12892 - Crash main.events_restart : MDEV-12236 - Server shutdown problem main.events_slowlog : MDEV-12821 - Wrong result -main.explain_slowquerylog : Modified in 10.3.7 -main.func_json : Modified in 10.3.8 +main.explain_slowquerylog : Modified in 10.2.16 +main.func_json : Modified in 10.3.9 main.func_misc : Modified in 10.3.8 -main.func_time : Modified in 10.3.8 main.func_time_hires : Modified in 10.3.8 +main.func_time : Modified in 10.3.9 main.gis : MDEV-13411 - wrong result on P8 -main.grant : Modified in 10.3.8 main.grant2 : Modified in 10.3.8 -main.grant_not_windows : Added in 10.3.7 +main.grant : Modified in 10.3.9 +main.grant_not_windows : Added in 10.2.16 main.having : Modified in 10.3.8 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout main.index_merge_innodb : MDEV-7142 - Plan mismatch main.innodb_mysql_lock : MDEV-7861 - Wrong result -main.insert_select : Modified in 10.3.7 -main.invisible_field_grant_completely : Added in 10.3.7 -main.invisible_field_grant_system : Added in 10.3.7 +main.insert_select : Modified in 10.2.16 +main.invisible_field_debug : Modified in 10.3.9 +main.join : Modified in 10.1.35, 10.2.17, 10.3.9 +main.join_cache : Modified in 10.1.35, 10.2.17, 10.3.9 +main.join_outer : Modified in 10.1.35, 10.2.17, 10.3.9 main.kill-2 : MDEV-13257 - Wrong result main.kill_processlist-6619 : MDEV-10793 - Wrong result main.limit : Modified in 10.3.8 @@ -328,39 +371,34 @@ main.lock : Modified in main.log_slow : MDEV-13263 - Wrong result main.log_tables-big : MDEV-13408 - wrong result main.max_statement_time : Modified in 10.3.8 +main.mdev375 : MDEV-10607 - sporadic "can't connect" main.mdev-504 : MDEV-15171 - warning main.merge : MDEV-10607 - sporadic "can't connect" -main.multi_update : Modified in 10.3.7 -main.mysql : Modified in 10.3.7 -main.mysql_client_test_comp : MDEV-16641 - exec failed +main.myisam : Modified in 10.0.36 +main.mysql_client_test_comp : MDEV-16641 - Error in exec main.mysql_client_test_nonblock : CONC-208 - Error on Power; MDEV-15096 - exec failed -main.mysql_cp932 : Modified in 10.3.7 -main.mysql_upgrade_noengine : MDEV-14355 - Wrong result -main.mysql_upgrade_ssl : MDEV-13492 - Unknown SSL error +main.mysql_cp932 : Modified in 10.2.16 main.mysqld_option_err : MDEV-12747 - Timeout -main.mysqldump : MDEV-14800 - Stack smashing detected; modified in 10.3.8 +main.mysqldump : MDEV-14800 - Stack smashing detected; modified in 10.2.16, 10.3.8 main.mysqlhotcopy_myisam : MDEV-10995 - Hang on debug +main.mysql : Modified in 10.2.16 main.mysqlslap : Modified in 10.3.8 main.mysqltest : MDEV-13887 - Wrong result -main.mysqltest_tracking_info : Added in 10.3.8 +main.mysqltest_tracking_info : Added in 10.2.17, 10.3.8 +main.mysql_upgrade_noengine : MDEV-14355 - Wrong result +main.mysql_upgrade_ssl : MDEV-13492 - Unknown SSL error main.olap : Modified in 10.3.8 main.openssl_1 : MDEV-13492 - Unknown SSL error main.order_by_optimizer_innodb : MDEV-10683 - Wrong result -main.parser : Modified in 10.3.7 main.partition_debug_sync : MDEV-15669 - Deadlock found when trying to get lock -main.partition_innodb : Modified in 10.3.7 main.partition_innodb_plugin : MDEV-12901 - Valgrind warnings -main.partition_open_files_limit : Modified in 10.3.8 -main.ps : MDEV-11017 - Wrong result; modified in 10.3.8 -main.query_cache : MDEV-16180 - Wrong result +main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count; modified in 10.1.34 main.query_cache_debug : MDEV-15281 - Query cache is disabled +main.query_cache : MDEV-16180 - Wrong result main.range_vs_index_merge_innodb : MDEV-15283 - Server has gone away main.read_only_innodb : Modified in 10.3.8 -main.rename : Modified in 10.3.8 +main.rename : Modified in 10.3.9 main.reset_connection : Added in 10.3.8 -main.select : MDEV-15430 - Wrong result with clang-4 (compiler bug) -main.select_jcl6 : MDEV-15430 - Wrong result with clang-4 (compiler bug) -main.select_pkeycache : MDEV-15430 - Wrong result with clang-4 (compiler bug) main.selectivity : Modified in 10.3.8 main.session_tracker_last_gtid : Added in 10.3.8 main.set_password : Added in 10.3.8 @@ -368,55 +406,55 @@ main.set_statement : MDEV-13183 main.shm : MDEV-12727 - Mismatch, ERROR 2013 main.show_explain : MDEV-10674 - Wrong result code main.show_grants_with_plugin-7985 : Modified in 10.3.8 -main.sp : MDEV-7866 - Mismatch; modified in 10.3.8 -main.sp-code : Modified in 10.3.7 main.sp-condition-handler : Added in 10.3.8 main.sp-cursor : Modified in 10.3.8 +main.sp-innodb : Modified in 10.0.36 +main.sp : MDEV-7866 - Mismatch; modified in 10.3.8 +main.sp_notembedded : MDEV-10607 - internal error main.sp-row : Modified in 10.3.8 main.sp-security : Modified in 10.3.8 -main.sp_notembedded : MDEV-10607 - internal error main.sql_mode : Modified in 10.3.8 main.ssl_ca : MDEV-10895 - SSL connection error on Power main.ssl_cert_verify : MDEV-13735 - Server crash main.ssl_connect : MDEV-13492 - Unknown SSL error main.ssl_timeout : MDEV-11244 - Crash -main.stat_tables_par : MDEV-13266 - Wrong result -main.stat_tables_par_innodb : MDEV-14155 - Wrong rounding -main.statistics : Modified in 10.3.8 main.statistics_close : Added in 10.3.8 +main.statistics : Modified in 10.3.8 +main.stat_tables : Modified in 10.1.35, 10.2.17, 10.3.9 +main.stat_tables_par_innodb : MDEV-14155 - Wrong rounding +main.stat_tables_par : MDEV-13266 - Wrong result main.status : MDEV-13255 - Wrong result -main.subselect-crash_15755 : Added in 10.3.7 +main.subselect4 : Modified in 10.0.36 main.subselect_innodb : MDEV-10614 - Wrong result -main.subselect_sj2_mat : Modified in 10.3.8 -main.symlink-aria-11902 : MDEV-15098 - error 40 from storage engine -main.symlink-myisam-11902 : MDEV-15098 - Error 40 from storage engine +main.subselect : Modified in 10.1.35, 10.2.17, 10.3.9 +main.subselect_sj2_mat : Modified in 10.1.35, 10.2.17, 10.3.9 +main.subselect_sj_mat : Modified in 10.1.35, 10.2.17, 10.3.9 +main.subselect_sj : Modified in 10.0.36 main.tc_heuristic_recover : MDEV-14189 - Wrong result main.trigger : Modified in 10.3.8 main.type_bit : Modified in 10.3.8 main.type_blob : MDEV-15195 - Wrong result; modified in 10.3.8 main.type_datetime : MDEV-14322 - wrong result -main.type_datetime_hires : MDEV-15430 - Wrong result with clang-4 (compiler bug) main.type_decimal : Modified in 10.3.8 -main.type_float : MDEV-15430 - Wrong result with clang-4 (compiler bug) main.type_int : Modified in 10.3.8 -main.type_time_hires : MDEV-15430 - Wrong result with clang-4 (compiler bug) -main.type_timestamp_hires : MDEV-15430 - Wrong result with clang-4 (compiler bug) +main.union : Modified in 10.1.35, 10.2.17, 10.3.9 main.userstat : MDEV-12904 - SSL errors main.win : Modified in 10.3.8 -main.xa : MDEV-11769 - lock wait timeout +main.xa : MDEV-11769 - lock wait timeout; modified in 10.3.9 + #----------------------------------------------------------------------- -maria.alter : Modified in 10.3.7 +maria.alter : Modified in 10.2.16 maria.insert_select : MDEV-12757 - Timeout maria.insert_select-7314 : MDEV-16492 - Timeout maria.lock : Modified in 10.3.8 maria.maria : MDEV-14430 - Extra warning; modified in 10.3.8 -maria.maria-autoinc : MDEV-16652 - Out of range; added in 10.3.8 +maria.maria-autoinc : Added in 10.3.8 #----------------------------------------------------------------------- -mariabackup.absolute_ibdata_paths : MDEV-16642 - Wrong result +mariabackup.absolute_ibdata_paths : MDEV-16571 - Wrong result mariabackup.apply-log-only : MDEV-14192 - Assertion failure mariabackup.apply-log-only-incr : MDEV-14192 - Assertion failure; modified in 10.3.8 mariabackup.backup_ssl : MDEV-14192 - Assertion failure @@ -424,34 +462,43 @@ mariabackup.data_directory : MDEV-15270 mariabackup.full_backup : MDEV-16571 - Wrong result mariabackup.huge_lsn : MDEV-15662 - Sequence number is in the future mariabackup.incremental_backup : MDEV-14192 - Assertion failure -mariabackup.incremental_encrypted : MDEV-14188 - Wrong result, MDEV-15667 - timeout +mariabackup.incremental_encrypted : MDEV-15667 - timeout; modified in 10.3.9 +mariabackup.innodb_log_optimize_ddl : Added in 10.2.17, 10.3.9 +mariabackup.log_checksum_mismatch : MDEV-16571 - Wrong result mariabackup.lock_ddl_per_table : Modified in 10.3.8 mariabackup.mdev-14447 : MDEV-15201 - Timeout; modified in 10.3.8 mariabackup.partial_exclude : MDEV-15270 - Error on exec mariabackup.partition_datadir : Modified in 10.3.8 mariabackup.rename_during_mdl_lock : Added in 10.3.8 +mariabackup.unsupported_redo : MDEV-14192 - Crash; modified in 10.2.17, 10.3.9 mariabackup.xb_aws_key_management : MDEV-15680 - Error: xtrabackup_copy_logfile() failed mariabackup.xb_compressed_encrypted : MDEV-14812 - Segmentation fault -mariabackup.xb_file_key_management : MDEV-16650 - Wrong result +mariabackup.xb_file_key_management : MDEV-16571 - Wrong result mariabackup.xb_history : MDEV-16268 - Error on exec mariabackup.xb_page_compress : MDEV-14810 - status: 1, errno: 11 -mariabackup.xb_partition : MDEV-16643 - Table does not exist mariabackup.xb_rocksdb : Added in 10.3.8 mariabackup.xb_rocksdb_datadir : Added in 10.3.8 mariabackup.xb_rocksdb_datadir_debug : Added in 10.3.8 +mariabackup.xb_partition : MDEV-14192 - Crash mariabackup.xbstream : MDEV-14192 - Crash #----------------------------------------------------------------------- -mroonga/storage.* : MDEV-16127 - Wrong result - mroonga/storage.column_datetime_32bit_2038 : Wrong result on Alpha mroonga/storage.column_datetime_32bit_before_unix_epoch : Wrong result on Alpha mroonga/storage.column_datetime_32bit_max : Wrong result on Alpha mroonga/storage.column_datetime_32bit_out_of_range : Wrong result on Alpha +mroonga/storage.index_multiple_column_range_all_used_less_than : MDEV-16127 - Wrong result with GCC 8 +mroonga/storage.index_multiple_column_range_all_used_less_than_or_equal : MDEV-16127 - Wrong result with GCC 8 +mroonga/storage.index_multiple_column_range_partially_used_have_prefix_less_than : MDEV-16127 - Wrong result with GCC 8 +mroonga/storage.index_multiple_column_range_partially_used_have_prefix_less_than_or_equal : MDEV-16127 - Wrong result with GCC 8 +mroonga/storage.index_multiple_column_range_partially_used_no_prefix_less_than : MDEV-16127 - Wrong result with GCC 8 +mroonga/storage.index_multiple_column_range_partially_used_no_prefix_less_than_or_equal : MDEV-16127 - Wrong result with GCC 8 mroonga/storage.index_multiple_column_unique_date_32bit_equal : Wrong result on Alpha mroonga/storage.index_multiple_column_unique_date_order_32bit_desc : Wrong result on Alpha mroonga/storage.index_multiple_column_unique_datetime_index_read : MDEV-8643 - Valgrind +mroonga/storage.optimization_order_limit_optimized_datetime_less_than : MDEV-16127 - Wrong result with GCC 8 +mroonga/storage.optimization_order_limit_optimized_datetime_less_than_or_equal : MDEV-16127 - Wrong result with GCC 8 mroonga/storage.repair_table_no_index_file : MDEV-9364 - wrong result, MDEV-14807 - wrong error message mroonga/wrapper.repair_table_no_index_file : MDEV-14807 - Wrong error message @@ -474,7 +521,8 @@ parts.partition_auto_increment_maria : MDEV-14430 parts.partition_debug_innodb : MDEV-10891 - Can't create UNIX socket; MDEV-15095 - Table doesn't exist parts.partition_exch_qa_10 : MDEV-11765 - wrong result parts.partition_innodb_status_file : MDEV-12901 - Valgrind -parts.show_create : Added in 10.3.7 +parts.partition_special_innodb : MDEV-16942 - Timeout +parts.truncate_locked : Added in 10.1.35, 10.2.17, 10.3.9 #----------------------------------------------------------------------- @@ -494,6 +542,7 @@ perfschema.hostcache_ipv6_addrinfo_bad_allow : MDEV-13260 perfschema.hostcache_ipv6_ssl : MDEV-10696 - Crash perfschema.partition : Added in 10.3.8 perfschema.privilege_table_io : MDEV-13184 - Extra lines +perfschema.rpl_gtid_func : MDEV-16897 - Wrong result perfschema.socket_connect : MDEV-15677 - Wrong result perfschema.socket_summary_by_event_name_func : MDEV-10622 - Wrong result perfschema.stage_mdl_global : MDEV-11803 - wrong result on slow builders @@ -520,20 +569,19 @@ rocksdb.2pc_group_commit : MDEV-14455 rocksdb.add_index_inplace : MDEV-16648 - Crash rocksdb.allow_no_primary_key : MDEV-16634 - Crash rocksdb.allow_no_primary_key_with_sk : MDEV-16639 - Crash -rocksdb.analyze_table : Modified in 10.3.7 +rocksdb.analyze_table : Modified in 10.2.16 rocksdb.autoinc_crash_safe : MDEV-16637 - Crash rocksdb.autoinc_crash_safe_partition : MDEV-16639 - Crash -rocksdb.autoinc_debug : MDEV-16203 - Wrong result rocksdb.autoinc_secondary : MDEV-16638 - Crash rocksdb.autoinc_vars_thread : MDEV-16573 - Debug sync timed out rocksdb.bloomfilter2 : MDEV-16564 - Wrong result rocksdb.bloomfilter4 : MDEV-16649 - Crash -rocksdb.bulk_load_errors : MDEV-16575 - Wrong result -rocksdb.check_ignore_unknown_options : MDEV-16310 - Non-portable commands; modified in 10.3.8 +rocksdb.check_ignore_unknown_options : Modified in 10.3.8 rocksdb.deadlock : MDEV-16033 - Timeout rocksdb.drop_index_inplace : MDEV-14162 - Crash on shutdown rocksdb.drop_table : MDEV-14308 - Timeout rocksdb.drop_table2 : MDEV-16631 - Crash +rocksdb.drop_table3 : MDEV-16949 - Server crash rocksdb.issue255 : MDEV-16577 - Wrong plan; modified in 10.3.8 rocksdb.locking_issues : MDEV-14464 - Wrong result rocksdb.mariadb_ignore_dirs : MDEV-16639 - Crash @@ -546,9 +594,12 @@ rocksdb.rocksdb_parts : MDEV-13843 rocksdb.singledelete : MDEV-16633 - Crash rocksdb.truncate_table3 : MDEV-14506 - Lost connection to server rocksdb.ttl_primary_read_filtering : MDEV-16560 - Wrong result +rocksdb.ttl_secondary : MDEV-16943 - Timeout rocksdb.ttl_secondary_read_filtering : MDEV-16560 - Wrong result rocksdb.unique_check : MDEV-16576 - Wrong errno rocksdb.use_direct_reads_writes : MDEV-16646 - Crash +rocksdb.validate_datadic : MDEV-12445 - Memory leak +rocksdb.write_sync : MDEV-16965 - Wrong result #----------------------------------------------------------------------- @@ -557,7 +608,10 @@ rocksdb_rpl.rpl_binlog_xid_count : MDEV-16644 #----------------------------------------------------------------------- +rocksdb_sys_vars.rocksdb_rate_limiter_bytes_per_sec_basic : MDEV-16639 - Crash rocksdb_sys_vars.rocksdb_remove_mariabackup_checkpoint_basic : Added in 10.3.8 +rocksdb_sys_vars.rocksdb_update_cf_options_basic : MDEV-16955 - Bytes lost +rocksdb_sys_vars.rocksdb_update_cf_options : MDEV-16955 - Bytes lost #----------------------------------------------------------------------- @@ -570,8 +624,11 @@ roles.set_default_role_ps-6960 : Modified in rpl-tokudb.* : MDEV-14354 - Tests fail with tcmalloc +rpl-tokudb.rpl_tokudb_commit_after_flush : MDEV-16966 - Server crash + rpl.last_insert_id : MDEV-10625 - warnings in error log rpl.rename : Added in 10.3.8 +rpl.rpl_15867 : Added in 10.2.17, 10.3.9 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log @@ -583,6 +640,7 @@ rpl.rpl_ddl : MDEV-10417 rpl.rpl_do_grant : Modified in 10.3.8 rpl.rpl_domain_id_filter_io_crash : MDEV-12729 - Timeout in include file, MDEV-13677 - Server crash rpl.rpl_domain_id_filter_restart : MDEV-10684 - Wrong result +rpl.rpl_drop_db_fail : MDEV-16898 - Slave fails to start rpl.rpl_extra_col_master_innodb : MDEV-16570 - Extra warning rpl.rpl_extra_col_master_myisam : MDEV-14203 - Extra warning rpl.rpl_gtid_basic : MDEV-10681 - server startup problem @@ -602,9 +660,8 @@ rpl.rpl_insert_id_pk : MDEV-16567 rpl.rpl_insert_ignore : MDEV-14365 - Lost connection to MySQL server during query rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips rpl.rpl_mariadb_slave_capability : MDEV-11018 - Extra lines in binlog -rpl.rpl_mdev12179 : Modified in 10.3.7 -rpl.rpl_mdev382 : Modified in 10.3.7 rpl.rpl_mdev6020 : MDEV-15272 - Server crash +rpl.rpl_mixed_implicit_commit_binlog : Included file modified in 10.0.36 rpl.rpl_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_non_direct_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_non_direct_row_mixing_engines : MDEV-16561 - Timeout in master_pos_wait @@ -619,13 +676,14 @@ rpl.rpl_parallel_retry : MDEV-11119 rpl.rpl_parallel_temptable : MDEV-10356 - Crash rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips rpl.rpl_password_boundaries : MDEV-11534 - Slave IO warnings -rpl.rpl_row_001 : MDEV-16653 - MTR's internal check fails +rpl.rpl_row_001 : MDEV-16653 - MTR's internal check fails; modified in 10.3.9 rpl.rpl_row_basic_11bugs : MDEV-12171 - Server failed to start rpl.rpl_row_basic_2myisam : MDEV-13875 - command "diff_files" failed rpl.rpl_row_drop_create_temp_table : MDEV-14487 - Wrong result rpl.rpl_row_img_blobs : MDEV-13875 - command "diff_files" failed rpl.rpl_row_img_eng_min : MDEV-13875 - diff_files failed rpl.rpl_row_img_eng_noblob : MDEV-13875 - command "diff_files" failed +rpl.rpl_row_implicit_commit_binlog : Included file modified in 10.0.36 rpl.rpl_row_index_choice : MDEV-15196 - Slave crash rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_row_until : MDEV-14052 - Master will not send events with checksum @@ -633,6 +691,7 @@ rpl.rpl_semi_sync : MDEV-11220 rpl.rpl_semi_sync_after_sync : MDEV-14366 - Wrong result rpl.rpl_semi_sync_after_sync_row : MDEV-14366 - Wrong result rpl.rpl_semi_sync_event_after_sync : MDEV-11806 - warnings +rpl.rpl_semi_sync_master_shutdown : Added in 10.3.9 rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Assorted failures rpl.rpl_semi_sync_wait_point : MDEV-11807 - timeout in wait condition rpl.rpl_semisync_ali_issues : MDEV-16272 - Wrong result @@ -644,7 +703,8 @@ rpl.rpl_slave_load_tmpdir_not_exist : MDEV-14203 rpl.rpl_slow_query_log : MDEV-13250 - Test abort rpl.rpl_sp_effects : MDEV-13249 - Crash rpl.rpl_start_stop_slave : MDEV-13567 - Sync slave timeout -rpl.rpl_stm_000001 : MDEV-16274 - Connection attributes were truncated +rpl.rpl_stm_000001 : MDEV-16274 - Connection attributes were truncated; modified in 10.3.9 +rpl.rpl_stm_implicit_commit_binlog : Included file modified in 10.0.36 rpl.rpl_stm_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_stm_multi_query : MDEV-9501 - Failed registering on master rpl.rpl_stm_relay_ign_space : MDEV-14360 - Test assertion @@ -669,6 +729,7 @@ sphinx.union-5539 : MDEV-10986 spider.* : MDEV-9329 - tests are too memory-consuming spider.basic_sql : MDEV-11186 - Internal check fails spider.spider_fixes_part : Modified in 10.3.8 +spider.timestamp : Added in 10.3.9 spider/bg.direct_aggregate : MDEV-7098 - Packets out of order spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked @@ -699,7 +760,7 @@ stress.ddl_innodb : MDEV-10635 sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x sys_vars.innodb_buffer_pool_dump_at_shutdown_basic : MDEV-14280 - Unexpected error -sys_vars.innodb_stats_include_delete_marked_basic : Modified in 10.3.7 +sys_vars.innodb_log_optimize_ddl_basic : Added in 10.2.17, 10.3.9 sys_vars.keep_files_on_create_basic : MDEV-10676 - timeout sys_vars.log_slow_admin_statements_func : MDEV-12235 - Server crash sys_vars.maximum_basic : Modified in 10.3.8 @@ -748,6 +809,7 @@ tokudb_bugs.frm_store : MDEV-12823 tokudb_bugs.frm_store2 : MDEV-12823 - Valgrind tokudb_bugs.frm_store3 : MDEV-12823 - Valgrind tokudb_bugs.xa : MDEV-11804 - Lock wait timeout +tokudb_bugs.xa-3 : MDEV-16953 - Corrupt log record found #----------------------------------------------------------------------- @@ -783,16 +845,13 @@ vcol.vcol_misc : MDEV-16651 #----------------------------------------------------------------------- versioning.alter : Modified in 10.3.8 -versioning.partition : Modified in 10.3.7 versioning.replace : Modified in 10.3.8 versioning.select : Modified in 10.3.8 -versioning.truncate : Modified in 10.3.7 versioning.trx_id : Modified in 10.3.8 versioning.trx_id_versioning_attribute_persistence : Added in 10.3.8 #----------------------------------------------------------------------- -wsrep.binlog_format : MDEV-11532 - Could not execute check-testcase wsrep.foreign_key : MDEV-14725 - WSREP has not yet prepared node wsrep.mdev_6832 : MDEV-14195 - Check testcase failed wsrep.pool_of_threads : MDEV-12234 - GLIBCXX_3.4.20 not found From dfb19c06b867786b5d9a9b49e7d10fc86c1d4057 Mon Sep 17 00:00:00 2001 From: mkaruza Date: Tue, 14 Aug 2018 10:34:51 +0200 Subject: [PATCH 023/111] MDEV-15933 Cannot resume Node SYNCED state when wsrep_desync is done after FTWRL Manually setting wsrep_desync after FTWRL should not be allowed. --- mysql-test/suite/galera/r/galera_var_desync_on.result | 2 -- mysql-test/suite/galera/t/galera_var_desync_on.test | 7 +++---- sql/wsrep_var.cc | 6 ++++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_desync_on.result b/mysql-test/suite/galera/r/galera_var_desync_on.result index f286ae72308..7d86555150e 100644 --- a/mysql-test/suite/galera/r/galera_var_desync_on.result +++ b/mysql-test/suite/galera/r/galera_var_desync_on.result @@ -1,7 +1,6 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); SET GLOBAL wsrep_provider_options = 'gcs.fc_limit=1'; -SET GLOBAL wsrep_desync = TRUE; FLUSH TABLES WITH READ LOCK; INSERT INTO t1 VALUES (2); INSERT INTO t1 VALUES (3); @@ -16,7 +15,6 @@ SET SESSION wsrep_sync_wait = 0; SELECT COUNT(*) = 1 FROM t1; COUNT(*) = 1 1 -SET GLOBAL wsrep_desync = FALSE; UNLOCK TABLES; SET SESSION wsrep_sync_wait = 1; SELECT COUNT(*) = 10 FROM t1; diff --git a/mysql-test/suite/galera/t/galera_var_desync_on.test b/mysql-test/suite/galera/t/galera_var_desync_on.test index 06c5d30a769..fbf660d3ab5 100644 --- a/mysql-test/suite/galera/t/galera_var_desync_on.test +++ b/mysql-test/suite/galera/t/galera_var_desync_on.test @@ -1,5 +1,7 @@ # -# Test wsrep_desync = ON . Node should temporarily not participate in flow control +# Desync will be done once the global read lock is acquired and resync will be done when +# it is released. +# Node should temporarily not participate in flow control # so even if fc_limit has been reached, the master should be able to continue to # commit transactions. # @@ -13,7 +15,6 @@ INSERT INTO t1 VALUES (1); --connection node_2 --let $wsrep_provider_options_orig = `SELECT @@wsrep_provider_options` SET GLOBAL wsrep_provider_options = 'gcs.fc_limit=1'; -SET GLOBAL wsrep_desync = TRUE; # Block the slave applier thread FLUSH TABLES WITH READ LOCK; @@ -37,8 +38,6 @@ SET SESSION wsrep_sync_wait = 0; # No updates have arrived after the FLUSH TABLES SELECT COUNT(*) = 1 FROM t1; -# Resync the slave -SET GLOBAL wsrep_desync = FALSE; --disable_query_log --eval SET GLOBAL wsrep_provider_options = '$wsrep_provider_options_orig'; --enable_query_log diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 216bab0cdcd..3826ebed14c 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -587,6 +587,12 @@ bool wsrep_desync_check (sys_var *self, THD* thd, set_var* var) return true; } + if (thd->global_read_lock.is_acquired()) + { + my_message (ER_CANNOT_USER, "Global read lock acquired. Can't set 'wsrep_desync'", MYF(0)); + return true; + } + bool new_wsrep_desync= (bool) var->save_result.ulonglong_value; if (wsrep_desync == new_wsrep_desync) { if (new_wsrep_desync) { From 68eb9b1a78423b9c0a353bc1bdfa7b518977dbac Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 14 Aug 2018 22:42:46 +0200 Subject: [PATCH 024/111] MDEV-16220 Do not pass UTF8 to mysql in command line parameters, on Windows Moved parts of mysql.test to mysql_not_windows.test --- mysql-test/r/mysql.result | 2 -- mysql-test/r/mysql_not_windows.result | 2 ++ mysql-test/t/mysql.test | 5 +---- mysql-test/t/mysql_not_windows.test | 7 +++++++ 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/mysql.result b/mysql-test/r/mysql.result index 09f014da627..06ef7e95bc9 100644 --- a/mysql-test/r/mysql.result +++ b/mysql-test/r/mysql.result @@ -538,8 +538,6 @@ a # # End of 10.1 tests # -ERROR 1300 (HY000): Invalid utf8 character string: 'test\xF0\x9F\x98\x81 ' -ERROR 1300 (HY000): Invalid binary character string: 'test\xF0\x9F\x98\x81 ' ERROR 1300 (HY000) at line 2: Invalid utf8 character string: 'test\xF0\x9F\x98\x81' set GLOBAL sql_mode=default; diff --git a/mysql-test/r/mysql_not_windows.result b/mysql-test/r/mysql_not_windows.result index 1df62d9a12d..96210a366a6 100644 --- a/mysql-test/r/mysql_not_windows.result +++ b/mysql-test/r/mysql_not_windows.result @@ -9,3 +9,5 @@ End of tests 2 X 3 +ERROR 1300 (HY000): Invalid utf8 character string: 'test\xF0\x9F\x98\x81 ' +ERROR 1300 (HY000): Invalid binary character string: 'test\xF0\x9F\x98\x81 ' diff --git a/mysql-test/t/mysql.test b/mysql-test/t/mysql.test index fc2ae849aa6..e3219c21517 100644 --- a/mysql-test/t/mysql.test +++ b/mysql-test/t/mysql.test @@ -638,10 +638,7 @@ EOF --echo # End of 10.1 tests --echo # ---error 1 ---exec $MYSQL --default-character-set=utf8 -e "select 1" "test😁 " 2>&1 ---error 1 ---exec $MYSQL --default-character-set=binary -e "select 1" "test😁 " 2>&1 + --write_file $MYSQLTEST_VARDIR/tmp/mdev-6572.sql SET NAMES utf8; USE test😁 ; diff --git a/mysql-test/t/mysql_not_windows.test b/mysql-test/t/mysql_not_windows.test index 591de74cbbf..816160c4f3e 100644 --- a/mysql-test/t/mysql_not_windows.test +++ b/mysql-test/t/mysql_not_windows.test @@ -22,3 +22,10 @@ exec $MYSQL test -e "select let $query = select 3 as X; exec $MYSQL test -e "$query"; + +# Not ran on Windows, since non-ASCII does not work on command line. +# (MDEV-16220) +--error 1 +--exec $MYSQL --default-character-set=utf8 -e "select 1" "test😁 " 2>&1 +--error 1 +--exec $MYSQL --default-character-set=binary -e "select 1" "test😁 " 2>&1 From 75dfd4acb995789ca5f86ccbd361fff9d2797e79 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 26 Jul 2018 15:04:11 +0200 Subject: [PATCH 025/111] This is patch for the https://jira.mariadb.org/browse/MDEV-9519 issue: If we have a 2+ node cluster which is replicating from an async master and the binlog_format is set to STATEMENT and multi-row inserts are executed on a table with an auto_increment column such that values are automatically generated by MySQL, then the server node generates wrong auto_increment values, which are different from what was generated on the async master. The causes and fixes: 1. We need to improve processing of changing the auto-increment values after changing the cluster size. 2. If wsrep auto_increment_control switched on during operation of the node, then we should immediately update the auto_increment_increment and auto_increment_offset global variables, without waiting of the next invocation of the wsrep_view_handler_cb() callback. In the current version these variables retain its initial values if wsrep_auto_increment_control is switched on during operation of the node, which leads to inconsistent results on the different nodes in some scenarios. 3. If wsrep auto_increment_control switched off during operation of the node, then we must return the original values of the auto_increment_increment and auto_increment_offset global variables, as the user has set. To make this possible, we need to add a "shadow copies" of these variables (which stores the latest values set by the user). --- .../r/galera_binlog_stmt_autoinc.result | 147 ++++++++++++ .../galera/t/galera_binlog_stmt_autoinc.test | 223 ++++++++++++++++++ sql/field.h | 46 ++++ sql/handler.cc | 31 ++- sql/mysqld.cc | 14 ++ sql/sql_class.h | 11 + sql/sys_vars.cc | 93 +++++++- sql/wsrep_mysqld.h | 4 + sql/wsrep_thd.cc | 22 ++ storage/innobase/handler/ha_innodb.cc | 59 +++-- storage/xtradb/handler/ha_innodb.cc | 57 ++++- 11 files changed, 677 insertions(+), 30 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result create mode 100644 mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.test diff --git a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result new file mode 100644 index 00000000000..542bd156816 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result @@ -0,0 +1,147 @@ +SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; +SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; +CREATE TABLE t1 ( +i int(11) NOT NULL AUTO_INCREMENT, +c char(32) DEFAULT 'dummy_text', +PRIMARY KEY (i) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +insert into t1(i) values(null); +select * from t1; +i c +1 dummy_text +insert into t1(i) values(null), (null), (null); +select * from t1; +i c +1 dummy_text +3 dummy_text +5 dummy_text +7 dummy_text +select * from t1; +i c +1 dummy_text +3 dummy_text +5 dummy_text +7 dummy_text +SET GLOBAL wsrep_forced_binlog_format='none'; +SET GLOBAL wsrep_forced_binlog_format='none'; +drop table t1; +SET SESSION binlog_format='STATEMENT'; +show variables like 'binlog_format'; +Variable_name Value +binlog_format STATEMENT +SET GLOBAL wsrep_auto_increment_control='OFF'; +SET SESSION auto_increment_increment = 3; +SET SESSION auto_increment_offset = 1; +CREATE TABLE t1 ( +i int(11) NOT NULL AUTO_INCREMENT, +c char(32) DEFAULT 'dummy_text', +PRIMARY KEY (i) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +insert into t1(i) values(null); +select * from t1; +i c +1 dummy_text +insert into t1(i) values(null), (null), (null); +select * from t1; +i c +1 dummy_text +4 dummy_text +7 dummy_text +10 dummy_text +select * from t1; +i c +1 dummy_text +4 dummy_text +7 dummy_text +10 dummy_text +SET GLOBAL wsrep_auto_increment_control='ON'; +SET SESSION binlog_format='ROW'; +show variables like 'binlog_format'; +Variable_name Value +binlog_format ROW +show variables like '%auto_increment%'; +Variable_name Value +auto_increment_increment 2 +auto_increment_offset 1 +wsrep_auto_increment_control ON +SET GLOBAL wsrep_auto_increment_control='OFF'; +show variables like '%auto_increment%'; +Variable_name Value +auto_increment_increment 3 +auto_increment_offset 1 +wsrep_auto_increment_control OFF +SET GLOBAL wsrep_auto_increment_control='ON'; +drop table t1; +SET GLOBAL wsrep_forced_binlog_format='ROW'; +SET GLOBAL wsrep_forced_binlog_format='ROW'; +CREATE TABLE t1 ( +i int(11) NOT NULL AUTO_INCREMENT, +c char(32) DEFAULT 'dummy_text', +PRIMARY KEY (i) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +insert into t1(i) values(null); +select * from t1; +i c +1 dummy_text +insert into t1(i) values(null), (null), (null); +select * from t1; +i c +1 dummy_text +3 dummy_text +5 dummy_text +7 dummy_text +select * from t1; +i c +1 dummy_text +3 dummy_text +5 dummy_text +7 dummy_text +SET GLOBAL wsrep_forced_binlog_format='none'; +SET GLOBAL wsrep_forced_binlog_format='none'; +drop table t1; +SET SESSION binlog_format='ROW'; +show variables like 'binlog_format'; +Variable_name Value +binlog_format ROW +SET GLOBAL wsrep_auto_increment_control='OFF'; +SET SESSION auto_increment_increment = 3; +SET SESSION auto_increment_offset = 1; +CREATE TABLE t1 ( +i int(11) NOT NULL AUTO_INCREMENT, +c char(32) DEFAULT 'dummy_text', +PRIMARY KEY (i) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +insert into t1(i) values(null); +select * from t1; +i c +1 dummy_text +insert into t1(i) values(null), (null), (null); +select * from t1; +i c +1 dummy_text +4 dummy_text +7 dummy_text +10 dummy_text +select * from t1; +i c +1 dummy_text +4 dummy_text +7 dummy_text +10 dummy_text +SET GLOBAL wsrep_auto_increment_control='ON'; +show variables like 'binlog_format'; +Variable_name Value +binlog_format ROW +show variables like '%auto_increment%'; +Variable_name Value +auto_increment_increment 2 +auto_increment_offset 1 +wsrep_auto_increment_control ON +SET GLOBAL wsrep_auto_increment_control='OFF'; +show variables like '%auto_increment%'; +Variable_name Value +auto_increment_increment 3 +auto_increment_offset 1 +wsrep_auto_increment_control OFF +SET GLOBAL wsrep_auto_increment_control='ON'; +drop table t1; diff --git a/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.test b/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.test new file mode 100644 index 00000000000..aab4fea9f2e --- /dev/null +++ b/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.test @@ -0,0 +1,223 @@ +## +## Tests the auto-increment with binlog in STATEMENT mode. +## + +--source include/galera_cluster.inc +--source include/have_innodb.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +## +## Verify the correct operation of the auto-increment when the binlog +## format artificially set to the 'STATEMENT' (although this mode is +## not recommended in the current version): +## + +--connection node_2 +SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; + +--connection node_1 +SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; + +CREATE TABLE t1 ( + i int(11) NOT NULL AUTO_INCREMENT, + c char(32) DEFAULT 'dummy_text', + PRIMARY KEY (i) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +insert into t1(i) values(null); + +select * from t1; + +insert into t1(i) values(null), (null), (null); + +select * from t1; + +--connection node_2 + +select * from t1; + +SET GLOBAL wsrep_forced_binlog_format='none'; + +--connection node_1 + +SET GLOBAL wsrep_forced_binlog_format='none'; + +drop table t1; + +## +## Check the operation when the automatic control over the auto-increment +## settings is switched off, that is, when we use the increment step and +## the offset specified by the user. In the current session, the binlog +## format is set to 'STATEMENT'. It is important that the values of the +## auto-increment options does not changed on other node - it allows us +## to check the correct transmission of the auto-increment options to +## other nodes: +## + +--disable_warnings +SET SESSION binlog_format='STATEMENT'; +--enable_warnings + +show variables like 'binlog_format'; + +SET GLOBAL wsrep_auto_increment_control='OFF'; + +SET SESSION auto_increment_increment = 3; +SET SESSION auto_increment_offset = 1; + +CREATE TABLE t1 ( + i int(11) NOT NULL AUTO_INCREMENT, + c char(32) DEFAULT 'dummy_text', + PRIMARY KEY (i) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +insert into t1(i) values(null); + +select * from t1; + +insert into t1(i) values(null), (null), (null); + +select * from t1; + +--connection node_2 + +select * from t1; + +--connection node_1 + +## +## Verify the return to automatic calculation of the step +## and offset of the auto-increment: +## + +SET GLOBAL wsrep_auto_increment_control='ON'; + +SET SESSION binlog_format='ROW'; + +show variables like 'binlog_format'; +show variables like '%auto_increment%'; + +## +## Verify the recovery of original user-defined values after +## stopping the automatic control over auto-increment: +## + +SET GLOBAL wsrep_auto_increment_control='OFF'; + +show variables like '%auto_increment%'; + +## +## Restore original options and drop test table: +## + +SET GLOBAL wsrep_auto_increment_control='ON'; + +drop table t1; + +## +## Verify the correct operation of the auto-increment when the binlog +## format set to the 'ROW': +## + +--connection node_2 +SET GLOBAL wsrep_forced_binlog_format='ROW'; + +--connection node_1 +SET GLOBAL wsrep_forced_binlog_format='ROW'; + +CREATE TABLE t1 ( + i int(11) NOT NULL AUTO_INCREMENT, + c char(32) DEFAULT 'dummy_text', + PRIMARY KEY (i) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +insert into t1(i) values(null); + +select * from t1; + +insert into t1(i) values(null), (null), (null); + +select * from t1; + +--connection node_2 + +select * from t1; + +SET GLOBAL wsrep_forced_binlog_format='none'; + +--connection node_1 + +SET GLOBAL wsrep_forced_binlog_format='none'; + +drop table t1; + +## +## Check the operation when the automatic control over the auto-increment +## settings is switched off, that is, when we use the increment step and +## the offset specified by the user. In the current session, the binlog +## format is set to 'ROW'. It is important that the values of the +## auto-increment options does not changed on other node - it allows us +## to check the correct transmission of the auto-increment options to +## other nodes: +## + +SET SESSION binlog_format='ROW'; + +show variables like 'binlog_format'; + +SET GLOBAL wsrep_auto_increment_control='OFF'; + +SET SESSION auto_increment_increment = 3; +SET SESSION auto_increment_offset = 1; + +CREATE TABLE t1 ( + i int(11) NOT NULL AUTO_INCREMENT, + c char(32) DEFAULT 'dummy_text', + PRIMARY KEY (i) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +insert into t1(i) values(null); + +select * from t1; + +insert into t1(i) values(null), (null), (null); + +select * from t1; + +--connection node_2 + +select * from t1; + +--connection node_1 + +## +## Verify the return to automatic calculation of the step +## and offset of the auto-increment: +## + +SET GLOBAL wsrep_auto_increment_control='ON'; + +show variables like 'binlog_format'; +show variables like '%auto_increment%'; + +## +## Verify the recovery of original user-defined values after +## stopping the automatic control over auto-increment: +## + +SET GLOBAL wsrep_auto_increment_control='OFF'; + +show variables like '%auto_increment%'; + +## +## Restore original options and drop test table: +## + +SET GLOBAL wsrep_auto_increment_control='ON'; + +drop table t1; + +--source include/auto_increment_offset_restore.inc diff --git a/sql/field.h b/sql/field.h index c21bba6936f..30b4b9fd68e 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1329,6 +1329,17 @@ public: /* Hash value */ virtual void hash(ulong *nr, ulong *nr2); + /** + Get the upper limit of the MySQL integral and floating-point type. + + @return maximum allowed value for the field + */ + virtual ulonglong get_max_int_value() const + { + DBUG_ASSERT(false); + return 0ULL; + } + /** Checks whether a string field is part of write_set. @@ -1796,6 +1807,11 @@ public: *to= *from; return from + 1; } + + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFULL : 0x7FULL; + } }; @@ -1837,6 +1853,10 @@ public: virtual const uchar *unpack(uchar* to, const uchar *from, const uchar *from_end, uint param_data) { return unpack_int16(to, from, from_end); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFULL : 0x7FFFULL; + } }; class Field_medium :public Field_num { @@ -1870,6 +1890,10 @@ public: { return Field::pack(to, from, max_length); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFULL : 0x7FFFFFULL; + } }; @@ -1915,6 +1939,10 @@ public: { return unpack_int32(to, from, from_end); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFFFULL : 0x7FFFFFFFULL; + } }; @@ -1964,6 +1992,10 @@ public: { return unpack_int64(to, from, from_end); } + virtual ulonglong get_max_int_value() const + { + return unsigned_flag ? 0xFFFFFFFFFFFFFFFFULL : 0x7FFFFFFFFFFFFFFFULL; + } }; @@ -1997,6 +2029,13 @@ public: uint32 pack_length() const { return sizeof(float); } uint row_pack_length() const { return pack_length(); } void sql_type(String &str) const; + virtual ulonglong get_max_int_value() const + { + /* + We use the maximum as per IEEE754-2008 standard, 2^24 + */ + return 0x1000000ULL; + } private: int do_save_field_metadata(uchar *first_byte); }; @@ -2039,6 +2078,13 @@ public: uint32 pack_length() const { return sizeof(double); } uint row_pack_length() const { return pack_length(); } void sql_type(String &str) const; + virtual ulonglong get_max_int_value() const + { + /* + We use the maximum as per IEEE754-2008 standard, 2^53 + */ + return 0x20000000000000ULL; + } private: int do_save_field_metadata(uchar *first_byte); }; diff --git a/sql/handler.cc b/sql/handler.cc index 9c319b995da..e2690023807 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -2853,9 +2853,15 @@ compute_next_insert_id(ulonglong nr,struct system_variables *variables) nr= nr + 1; // optimization of the formula below else { + /* + Calculating the number of complete auto_increment_increment extents: + */ nr= (((nr+ variables->auto_increment_increment - variables->auto_increment_offset)) / (ulonglong) variables->auto_increment_increment); + /* + Adding an offset to the auto_increment_increment extent boundary: + */ nr= (nr* (ulonglong) variables->auto_increment_increment + variables->auto_increment_offset); } @@ -2911,8 +2917,14 @@ prev_insert_id(ulonglong nr, struct system_variables *variables) } if (variables->auto_increment_increment == 1) return nr; // optimization of the formula below + /* + Calculating the number of complete auto_increment_increment extents: + */ nr= (((nr - variables->auto_increment_offset)) / (ulonglong) variables->auto_increment_increment); + /* + Adding an offset to the auto_increment_increment extent boundary: + */ return (nr * (ulonglong) variables->auto_increment_increment + variables->auto_increment_offset); } @@ -3134,10 +3146,23 @@ int handler::update_auto_increment() if (unlikely(tmp)) // Out of range value in store { /* - It's better to return an error here than getting a confusing - 'duplicate key error' later. + first test if the query was aborted due to strict mode constraints */ - result= HA_ERR_AUTOINC_ERANGE; + if (thd->killed == KILL_BAD_DATA || + nr > table->next_number_field->get_max_int_value()) + DBUG_RETURN(HA_ERR_AUTOINC_ERANGE); + + /* + field refused this value (overflow) and truncated it, use the result of + the truncation (which is going to be inserted); however we try to + decrease it to honour auto_increment_* variables. + That will shift the left bound of the reserved interval, we don't + bother shifting the right bound (anyway any other value from this + interval will cause a duplicate key). + */ + nr= prev_insert_id(table->next_number_field->val_int(), variables); + if (unlikely(table->next_number_field->store((longlong) nr, TRUE))) + nr= table->next_number_field->val_int(); } if (append) { diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 1e6e144ccff..d12cdb34e02 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4193,6 +4193,20 @@ static int init_common_variables() DBUG_PRINT("info",("%s Ver %s for %s on %s\n",my_progname, server_version, SYSTEM_TYPE,MACHINE_TYPE)); +#ifdef WITH_WSREP + /* + We need to initialize auxiliary variables, that will be + further keep the original values of auto-increment options + as they set by the user. These variables used to restore + user-defined values of the auto-increment options after + setting of the wsrep_auto_increment_control to 'OFF'. + */ + global_system_variables.saved_auto_increment_increment= + global_system_variables.auto_increment_increment; + global_system_variables.saved_auto_increment_offset= + global_system_variables.auto_increment_offset; +#endif /* WITH_WSREP */ + #ifdef HAVE_LARGE_PAGES /* Initialize large page size */ if (opt_large_pages) diff --git a/sql/sql_class.h b/sql/sql_class.h index 53e451511d5..f0c486bd172 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -552,6 +552,17 @@ typedef struct system_variables ha_rows max_join_size; ha_rows expensive_subquery_limit; ulong auto_increment_increment, auto_increment_offset; +#ifdef WITH_WSREP + /* + Variables with stored values of the auto_increment_increment + and auto_increment_offset options that are will be needed when + wsrep_auto_increment_control will be set to 'OFF', because the + setting it to 'ON' leads to overwriting of the original values + (which are set by the user) by calculated values (which are + based on the cluster's size): + */ + ulong saved_auto_increment_increment, saved_auto_increment_offset; +#endif /* WITH_WSREP */ ulong lock_wait_timeout; ulong join_cache_level; ulong max_allowed_packet; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index f941794f89e..8a67c1d38f3 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -345,13 +345,56 @@ static Sys_var_long Sys_pfs_connect_attrs_size( #endif /* WITH_PERFSCHEMA_STORAGE_ENGINE */ +#ifdef WITH_WSREP + +/* + We need to keep the original values set by the user, as they will + be lost if wsrep_auto_increment_control set to 'ON': +*/ +static bool update_auto_increment_increment (sys_var *self, THD *thd, enum_var_type type) +{ + if (type == OPT_GLOBAL) + global_system_variables.saved_auto_increment_increment= + global_system_variables.auto_increment_increment; + else + thd->variables.saved_auto_increment_increment= + thd->variables.auto_increment_increment; + return false; +} + +#endif /* WITH_WSREP */ + static Sys_var_ulong Sys_auto_increment_increment( "auto_increment_increment", "Auto-increment columns are incremented by this", SESSION_VAR(auto_increment_increment), CMD_LINE(OPT_ARG), VALID_RANGE(1, 65535), DEFAULT(1), BLOCK_SIZE(1), +#ifdef WITH_WSREP + NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_auto_increment_increment)); +#else NO_MUTEX_GUARD, IN_BINLOG); +#endif /* WITH_WSREP */ + +#ifdef WITH_WSREP + +/* + We need to keep the original values set by the user, as they will + be lost if wsrep_auto_increment_control set to 'ON': +*/ +static bool update_auto_increment_offset (sys_var *self, THD *thd, enum_var_type type) +{ + if (type == OPT_GLOBAL) + global_system_variables.saved_auto_increment_offset= + global_system_variables.auto_increment_offset; + else + thd->variables.saved_auto_increment_offset= + thd->variables.auto_increment_offset; + return false; +} + +#endif /* WITH_WSREP */ static Sys_var_ulong Sys_auto_increment_offset( "auto_increment_offset", @@ -360,7 +403,12 @@ static Sys_var_ulong Sys_auto_increment_offset( SESSION_VAR(auto_increment_offset), CMD_LINE(OPT_ARG), VALID_RANGE(1, 65535), DEFAULT(1), BLOCK_SIZE(1), +#ifdef WITH_WSREP + NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_auto_increment_offset)); +#else NO_MUTEX_GUARD, IN_BINLOG); +#endif /* WITH_WSREP */ static Sys_var_mybool Sys_automatic_sp_privileges( "automatic_sp_privileges", @@ -4847,11 +4895,54 @@ static Sys_var_ulong Sys_wsrep_retry_autocommit( SESSION_VAR(wsrep_retry_autocommit), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 10000), DEFAULT(1), BLOCK_SIZE(1)); +static bool update_wsrep_auto_increment_control (sys_var *self, THD *thd, enum_var_type type) +{ + if (wsrep_auto_increment_control) + { + /* + The variables that control auto increment shall be calculated + automaticaly based on the size of the cluster. This usually done + within the wsrep_view_handler_cb callback. However, if the user + manually sets the value of wsrep_auto_increment_control to 'ON', + then we should to re-calculate these variables again (because + these values may be required before wsrep_view_handler_cb will + be re-invoked, which is rarely invoked if the cluster stays in + the stable state): + */ + global_system_variables.auto_increment_increment= + wsrep_cluster_size ? wsrep_cluster_size : 1; + global_system_variables.auto_increment_offset= + wsrep_local_index >= 0 ? wsrep_local_index + 1 : 1; + thd->variables.auto_increment_increment= + global_system_variables.auto_increment_increment; + thd->variables.auto_increment_offset= + global_system_variables.auto_increment_offset; + } + else + { + /* + We must restore the last values of the variables that + are explicitly specified by the user: + */ + global_system_variables.auto_increment_increment= + global_system_variables.saved_auto_increment_increment; + global_system_variables.auto_increment_offset= + global_system_variables.saved_auto_increment_offset; + thd->variables.auto_increment_increment= + thd->variables.saved_auto_increment_increment; + thd->variables.auto_increment_offset= + thd->variables.saved_auto_increment_offset; + } + return false; +} + static Sys_var_mybool Sys_wsrep_auto_increment_control( "wsrep_auto_increment_control", "To automatically control the " "assignment of autoincrement variables", GLOBAL_VAR(wsrep_auto_increment_control), - CMD_LINE(OPT_ARG), DEFAULT(TRUE)); + CMD_LINE(OPT_ARG), DEFAULT(TRUE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(update_wsrep_auto_increment_control)); static Sys_var_mybool Sys_wsrep_drupal_282555_workaround( "wsrep_drupal_282555_workaround", "Enable a workaround to handle the " diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 272e4ac4984..cd1a8882a95 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -161,6 +161,10 @@ extern "C" query_id_t wsrep_thd_query_id(THD *thd); extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd); extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id); +extern "C" void wsrep_thd_auto_increment_variables(THD*, + unsigned long long *offset, + unsigned long long *increment); + extern void wsrep_close_client_connections(my_bool wait_to_end); extern int wsrep_wait_committing_connections_close(int wait_time); extern void wsrep_close_applier(THD *thd); diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc index eb26da61282..cd7359160d6 100644 --- a/sql/wsrep_thd.cc +++ b/sql/wsrep_thd.cc @@ -676,3 +676,25 @@ bool wsrep_thd_has_explicit_locks(THD *thd) assert(thd); return thd->mdl_context.has_explicit_locks(); } + +/* + Get auto increment variables for THD. Use global settings for + applier threads. + */ +extern "C" +void wsrep_thd_auto_increment_variables(THD* thd, + unsigned long long* offset, + unsigned long long* increment) +{ + if (thd->wsrep_exec_mode == REPL_RECV && + thd->wsrep_conflict_state != REPLAYING) + { + *offset= global_system_variables.auto_increment_offset; + *increment= global_system_variables.auto_increment_increment; + } + else + { + *offset= thd->variables.auto_increment_offset; + *increment= thd->variables.auto_increment_increment; + } +} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5667efea5df..3d569a2d1d7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -8396,14 +8396,29 @@ set_max_autoinc: /* This should filter out the negative values set explicitly by the user. */ if (auto_inc <= col_max_value) { - ut_a(prebuilt->autoinc_increment > 0); - ulonglong offset; ulonglong increment; dberr_t err; - offset = prebuilt->autoinc_offset; - increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + /* Applier threads which are + processing ROW events and don't go + through server level autoinc + processing, therefore m_prebuilt + autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(current_thd) && + wsrep_thd_exec_mode(current_thd) == REPL_RECV) { + wsrep_thd_auto_increment_variables(current_thd, &offset, &increment); + } else { +#endif /* WITH_WSREP */ + ut_a(prebuilt->autoinc_increment > 0); + offset = prebuilt->autoinc_offset; + increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, @@ -8918,16 +8933,32 @@ ha_innobase::update_row( /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - col_max_value = innobase_get_int_col_max_value( - table->next_number_field); + col_max_value = + table->next_number_field->get_max_int_value(); if (auto_inc <= col_max_value && auto_inc != 0) { ulonglong offset; ulonglong increment; - offset = prebuilt->autoinc_offset; - increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + m_prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(current_thd) && + wsrep_thd_exec_mode(current_thd) == REPL_RECV) { + wsrep_thd_auto_increment_variables( + current_thd, &offset, &increment); + } else { +#endif /* WITH_WSREP */ + offset = prebuilt->autoinc_offset; + increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, 1, increment, offset, col_max_value); @@ -16038,13 +16069,13 @@ ha_innobase::get_auto_increment( increment, thd_get_thread_id(ha_thd()), current, autoinc); - if (!wsrep_on(ha_thd())) - { - current = autoinc - prebuilt->autoinc_increment; - } - current = innobase_next_autoinc( - current, 1, increment, offset, col_max_value); + if (!wsrep_on(ha_thd())) { + current = autoinc - prebuilt->autoinc_increment; + + current = innobase_next_autoinc( + current, 1, increment, offset, col_max_value); + } dict_table_autoinc_initialize(prebuilt->table, current); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 3cd7cb6977b..857fc7e66f1 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -8988,8 +8988,25 @@ set_max_autoinc: ulonglong increment; dberr_t err; - offset = prebuilt->autoinc_offset; - increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + /* Applier threads which are + processing ROW events and don't go + through server level autoinc + processing, therefore m_prebuilt + autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(current_thd) && + wsrep_thd_exec_mode(current_thd) == REPL_RECV) { + wsrep_thd_auto_increment_variables(current_thd, &offset, &increment); + } else { +#endif /* WITH_WSREP */ + ut_a(prebuilt->autoinc_increment > 0); + offset = prebuilt->autoinc_offset; + increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, @@ -9502,16 +9519,32 @@ ha_innobase::update_row( /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ - col_max_value = innobase_get_int_col_max_value( - table->next_number_field); + col_max_value = + table->next_number_field->get_max_int_value(); if (auto_inc <= col_max_value && auto_inc != 0) { ulonglong offset; ulonglong increment; - offset = prebuilt->autoinc_offset; - increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + /* Applier threads which are processing + ROW events and don't go through server + level autoinc processing, therefore + m_prebuilt autoinc values don't get + properly assigned. Fetch values from + server side. */ + if (wsrep_on(current_thd) && + wsrep_thd_exec_mode(current_thd) == REPL_RECV) { + wsrep_thd_auto_increment_variables( + current_thd, &offset, &increment); + } else { +#endif /* WITH_WSREP */ + offset = prebuilt->autoinc_offset; + increment = prebuilt->autoinc_increment; +#ifdef WITH_WSREP + } +#endif /* WITH_WSREP */ auto_inc = innobase_next_autoinc( auto_inc, 1, increment, offset, col_max_value); @@ -16742,13 +16775,13 @@ ha_innobase::get_auto_increment( increment, thd_get_thread_id(ha_thd()), current, autoinc); - if (!wsrep_on(ha_thd())) - { - current = autoinc - prebuilt->autoinc_increment; - } - current = innobase_next_autoinc( - current, 1, increment, offset, col_max_value); + if (!wsrep_on(ha_thd())) { + current = autoinc - prebuilt->autoinc_increment; + + current = innobase_next_autoinc( + current, 1, increment, offset, col_max_value); + } dict_table_autoinc_initialize(prebuilt->table, current); From 1b797e9e6308913c2472f3e04ad253e95a35d59f Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Mon, 6 Aug 2018 15:50:22 +0200 Subject: [PATCH 026/111] MDEV-15475: Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' failed on EXPLAIN EXTENDED with constant table and view Print constant ISNULL value independent. Fix of printing of view FRM and CREATE VIEW output --- mysql-test/r/derived_view.result | 6 +++--- mysql-test/r/func_isnull.result | 20 ++++++++++++++++++++ mysql-test/r/subselect_mat.result | 6 +++--- mysql-test/r/subselect_sj_mat.result | 6 +++--- mysql-test/t/func_isnull.test | 16 ++++++++++++++++ sql/item.cc | 2 +- sql/item_cmpfunc.cc | 13 +++++++++++++ sql/item_cmpfunc.h | 1 + sql/sql_lex.cc | 2 +- sql/sql_show.cc | 2 +- 10 files changed, 62 insertions(+), 12 deletions(-) diff --git a/mysql-test/r/derived_view.result b/mysql-test/r/derived_view.result index f7062473a3f..12811ebc6b3 100644 --- a/mysql-test/r/derived_view.result +++ b/mysql-test/r/derived_view.result @@ -1101,7 +1101,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t3 system NULL NULL NULL NULL 1 100.00 2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select 6 AS `a`,5 AS `b` from `test`.`t3` where (not(<6,5>(((6,5),(select 7,5 having (trigcond((((6) = 7) or isnull(7))) and trigcond((((5) = 5) or isnull(5))) and trigcond((7)) and trigcond((5)))))))) +Note 1003 select 6 AS `a`,5 AS `b` from `test`.`t3` where (not(<6,5>(((6,5),(select 7,5 having (trigcond((((6) = 7) or isnull(/*always not null*/ 1))) and trigcond((((5) = 5) or isnull(/*always not null*/ 1))) and trigcond((7)) and trigcond((5)))))))) SELECT t.a,t.b FROM t3 RIGHT JOIN ((SELECT * FROM t1) AS t, t2) ON t2.b != 0 WHERE (t.a,t.b) NOT IN (SELECT 7, 5); a b @@ -1115,7 +1115,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t3 system NULL NULL NULL NULL 1 100.00 3 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select 6 AS `a`,5 AS `b` from `test`.`t3` where (not(<6,5>(((6,5),(select 7,5 having (trigcond((((6) = 7) or isnull(7))) and trigcond((((5) = 5) or isnull(5))) and trigcond((7)) and trigcond((5)))))))) +Note 1003 select 6 AS `a`,5 AS `b` from `test`.`t3` where (not(<6,5>(((6,5),(select 7,5 having (trigcond((((6) = 7) or isnull(/*always not null*/ 1))) and trigcond((((5) = 5) or isnull(/*always not null*/ 1))) and trigcond((7)) and trigcond((5)))))))) SELECT t.a,t.b FROM t3 RIGHT JOIN (v1 AS t, t2) ON t2.b != 0 WHERE (t.a,t.b) NOT IN (SELECT 7, 5); a b @@ -1129,7 +1129,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t3 system NULL NULL NULL NULL 1 100.00 2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select 6 AS `a`,5 AS `b` from `test`.`t3` where (not(<6,5>(((6,5),(select 7,5 having (trigcond((((6) = 7) or isnull(7))) and trigcond((((5) = 5) or isnull(5))) and trigcond((7)) and trigcond((5)))))))) +Note 1003 select 6 AS `a`,5 AS `b` from `test`.`t3` where (not(<6,5>(((6,5),(select 7,5 having (trigcond((((6) = 7) or isnull(/*always not null*/ 1))) and trigcond((((5) = 5) or isnull(/*always not null*/ 1))) and trigcond((7)) and trigcond((5)))))))) DROP VIEW v1; DROP TABLE t1,t2,t3; # diff --git a/mysql-test/r/func_isnull.result b/mysql-test/r/func_isnull.result index 88c5bfd5468..a97d4a67939 100644 --- a/mysql-test/r/func_isnull.result +++ b/mysql-test/r/func_isnull.result @@ -106,5 +106,25 @@ Note 1003 select `test`.`t2`.`d1` AS `d1`,`test`.`t1`.`d1` AS `d1` from `test`.` DROP VIEW v1; DROP TABLE t1,t2; # +# MDEV-15475: Assertion `!table || (!table->read_set || +# bitmap_is_set(table->read_set, field_index))' +# failed on EXPLAIN EXTENDED with constant table and view +# +CREATE TABLE t1 (pk INT PRIMARY KEY) ENGINE=MyISAM; +CREATE VIEW v1 AS SELECT * FROM t1; +INSERT INTO t1 VALUES (1); +EXPLAIN EXTENDED SELECT ISNULL(pk) FROM v1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 system NULL NULL NULL NULL 1 100.00 +Warnings: +Note 1003 select isnull(/*always not null*/ 1) AS `ISNULL(pk)` from dual +EXPLAIN EXTENDED SELECT IFNULL(pk,0) FROM v1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 system NULL NULL NULL NULL 1 100.00 +Warnings: +Note 1003 select ifnull(1,0) AS `IFNULL(pk,0)` from dual +DROP VIEW v1; +DROP TABLE t1; +# # End of 5.5 tests # diff --git a/mysql-test/r/subselect_mat.result b/mysql-test/r/subselect_mat.result index eca3b760b65..efc348a26ce 100644 --- a/mysql-test/r/subselect_mat.result +++ b/mysql-test/r/subselect_mat.result @@ -499,7 +499,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where 2 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <`test`.`t1`.`a1`,`test`.`t1`.`a2`>(((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(select '1 - 01','2 - 01' having ((((`test`.`t1`.`a1`) = '1 - 01') or isnull('1 - 01')) and (((`test`.`t1`.`a2`) = '2 - 01') or isnull('2 - 01')) and ('1 - 01') and ('2 - 01'))))) +Note 1003 select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <`test`.`t1`.`a1`,`test`.`t1`.`a2`>(((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(select '1 - 01','2 - 01' having ((((`test`.`t1`.`a1`) = '1 - 01') or isnull(/*always not null*/ 1)) and (((`test`.`t1`.`a2`) = '2 - 01') or isnull(/*always not null*/ 1)) and ('1 - 01') and ('2 - 01'))))) select * from t1 where (a1, a2) in (select '1 - 01', '2 - 01'); a1 a2 1 - 01 2 - 01 @@ -509,7 +509,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where 2 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <`test`.`t1`.`a1`,`test`.`t1`.`a2`>(((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(select '1 - 01','2 - 01' having ((((`test`.`t1`.`a1`) = '1 - 01') or isnull('1 - 01')) and (((`test`.`t1`.`a2`) = '2 - 01') or isnull('2 - 01')) and ('1 - 01') and ('2 - 01'))))) +Note 1003 select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <`test`.`t1`.`a1`,`test`.`t1`.`a2`>(((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(select '1 - 01','2 - 01' having ((((`test`.`t1`.`a1`) = '1 - 01') or isnull(/*always not null*/ 1)) and (((`test`.`t1`.`a2`) = '2 - 01') or isnull(/*always not null*/ 1)) and ('1 - 01') and ('2 - 01'))))) select * from t1 where (a1, a2) in (select '1 - 01', '2 - 01' from dual); a1 a2 1 - 01 2 - 01 @@ -1896,7 +1896,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where; Using join buffer (flat, BNL join) 2 MATERIALIZED t2 ALL NULL NULL NULL NULL 3 100.00 Warnings: -Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t2`.`c`) from `test`.`t2`) join `test`.`t1` where ((`test`.`t1`.`b` = 7) and (`test`.`t1`.`a` = ``.`MAX(c)`) and ((isnull(``.`MAX(c)`)) or (``.`MAX(c)` = 7))) +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t2`.`c`) from `test`.`t2`) join `test`.`t1` where ((`test`.`t1`.`b` = 7) and (`test`.`t1`.`a` = ``.`MAX(c)`) and ((isnull(/*always not null*/ 1)) or (``.`MAX(c)` = 7))) SELECT * FROM t1 WHERE a IN (SELECT MAX(c) FROM t2) AND b=7 AND (a IS NULL OR a=b); a b diff --git a/mysql-test/r/subselect_sj_mat.result b/mysql-test/r/subselect_sj_mat.result index 180c182a51a..fd9435e8a39 100644 --- a/mysql-test/r/subselect_sj_mat.result +++ b/mysql-test/r/subselect_sj_mat.result @@ -520,7 +520,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where 2 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <`test`.`t1`.`a1`,`test`.`t1`.`a2`>(((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(select '1 - 01','2 - 01' having ((((`test`.`t1`.`a1`) = '1 - 01') or isnull('1 - 01')) and (((`test`.`t1`.`a2`) = '2 - 01') or isnull('2 - 01')) and ('1 - 01') and ('2 - 01'))))) +Note 1003 select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <`test`.`t1`.`a1`,`test`.`t1`.`a2`>(((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(select '1 - 01','2 - 01' having ((((`test`.`t1`.`a1`) = '1 - 01') or isnull(/*always not null*/ 1)) and (((`test`.`t1`.`a2`) = '2 - 01') or isnull(/*always not null*/ 1)) and ('1 - 01') and ('2 - 01'))))) select * from t1 where (a1, a2) in (select '1 - 01', '2 - 01'); a1 a2 1 - 01 2 - 01 @@ -530,7 +530,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where 2 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <`test`.`t1`.`a1`,`test`.`t1`.`a2`>(((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(select '1 - 01','2 - 01' having ((((`test`.`t1`.`a1`) = '1 - 01') or isnull('1 - 01')) and (((`test`.`t1`.`a2`) = '2 - 01') or isnull('2 - 01')) and ('1 - 01') and ('2 - 01'))))) +Note 1003 select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <`test`.`t1`.`a1`,`test`.`t1`.`a2`>(((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(select '1 - 01','2 - 01' having ((((`test`.`t1`.`a1`) = '1 - 01') or isnull(/*always not null*/ 1)) and (((`test`.`t1`.`a2`) = '2 - 01') or isnull(/*always not null*/ 1)) and ('1 - 01') and ('2 - 01'))))) select * from t1 where (a1, a2) in (select '1 - 01', '2 - 01' from dual); a1 a2 1 - 01 2 - 01 @@ -1934,7 +1934,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where; Using join buffer (flat, BNL join) 2 MATERIALIZED t2 ALL NULL NULL NULL NULL 3 100.00 Warnings: -Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t2`.`c`) from `test`.`t2`) join `test`.`t1` where ((`test`.`t1`.`b` = 7) and (`test`.`t1`.`a` = ``.`MAX(c)`) and ((isnull(``.`MAX(c)`)) or (``.`MAX(c)` = 7))) +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t2`.`c`) from `test`.`t2`) join `test`.`t1` where ((`test`.`t1`.`b` = 7) and (`test`.`t1`.`a` = ``.`MAX(c)`) and ((isnull(/*always not null*/ 1)) or (``.`MAX(c)` = 7))) SELECT * FROM t1 WHERE a IN (SELECT MAX(c) FROM t2) AND b=7 AND (a IS NULL OR a=b); a b diff --git a/mysql-test/t/func_isnull.test b/mysql-test/t/func_isnull.test index 4c59fa3cbe8..7d1a7e83a1a 100644 --- a/mysql-test/t/func_isnull.test +++ b/mysql-test/t/func_isnull.test @@ -83,6 +83,22 @@ SELECT * FROM t2 LEFT JOIN v1 ON t2.d1=v1.d1 WHERE v1.d1 IS NULL; DROP VIEW v1; DROP TABLE t1,t2; + +--echo # +--echo # MDEV-15475: Assertion `!table || (!table->read_set || +--echo # bitmap_is_set(table->read_set, field_index))' +--echo # failed on EXPLAIN EXTENDED with constant table and view +--echo # + +CREATE TABLE t1 (pk INT PRIMARY KEY) ENGINE=MyISAM; +CREATE VIEW v1 AS SELECT * FROM t1; +INSERT INTO t1 VALUES (1); +EXPLAIN EXTENDED SELECT ISNULL(pk) FROM v1; +EXPLAIN EXTENDED SELECT IFNULL(pk,0) FROM v1; +# Cleanup +DROP VIEW v1; +DROP TABLE t1; + --echo # --echo # End of 5.5 tests --echo # diff --git a/sql/item.cc b/sql/item.cc index 33c35f8c3e0..0cf4864326f 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -6845,7 +6845,7 @@ Item *Item_field::update_value_transformer(uchar *select_arg) void Item_field::print(String *str, enum_query_type query_type) { if (field && field->table->const_table && - !(query_type & QT_NO_DATA_EXPANSION)) + !(query_type & (QT_NO_DATA_EXPANSION | QT_VIEW_INTERNAL))) { print_value(str); return; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 6fb650b975b..d4a2c767b15 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4850,6 +4850,19 @@ Item *and_expressions(Item *a, Item *b, Item **org_item) } +void Item_func_isnull::print(String *str, enum_query_type query_type) +{ + str->append(func_name()); + str->append('('); + if (const_item() && !args[0]->maybe_null && + !(query_type & (QT_NO_DATA_EXPANSION | QT_VIEW_INTERNAL))) + str->append("/*always not null*/ 1"); + else + args[0]->print(str, query_type); + str->append(')'); +} + + longlong Item_func_isnull::val_int() { DBUG_ASSERT(fixed == 1); diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 3c8cc71370d..c4e6a53dd6b 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1389,6 +1389,7 @@ public: const_item_cache= args[0]->const_item(); } } + virtual void print(String *str, enum_query_type query_type); table_map not_null_tables() const { return 0; } optimize_type select_optimize() const { return OPTIMIZE_NULL; } Item *neg_transformer(THD *thd); diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 2bf1216ec34..cfbde25314b 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -2414,7 +2414,7 @@ void st_select_lex::print_order(String *str, { if (order->counter_used) { - if (query_type != QT_VIEW_INTERNAL) + if (!(query_type & QT_VIEW_INTERNAL)) { char buffer[20]; size_t length= my_snprintf(buffer, 20, "%d", order->counter); diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 06d5a6f570a..db33a9de781 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2085,7 +2085,7 @@ view_store_create_info(THD *thd, TABLE_LIST *table, String *buff) We can't just use table->query, because our SQL_MODE may trigger a different syntax, like when ANSI_QUOTES is defined. */ - table->view->unit.print(buff, QT_ORDINARY); + table->view->unit.print(buff, QT_VIEW_INTERNAL); if (table->with_check != VIEW_CHECK_NONE) { From b62ac161856570e9a0e92d17de1e3dd31d54410f Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 15 Aug 2018 15:21:37 +0300 Subject: [PATCH 027/111] MDEV-6439: Server crashes in Explain_union::print_explain with explain in slow log, tis620 charset Item_subselect::is_expensive() used to return FALSE (Inexpensive) whenever it saw that one of SELECTs in the Subquery's UNION is degenerate. It ignored the fact that other parts of the UNION might not be inexpensive, including the case where pther parts of the UNION have no query plan yet. For a subquery in form col >= ANY (SELECT 'foo' UNION SELECT 'bar') this would cause the query to be considered inexpensive when there is no query plan for the second part of the UNION, which in turn would cause the SELECT 'foo' to compute and free itself while still inside JOIN::optimize for that SELECT (See MDEV comment for full description). --- .../r/subselect_extra_no_semijoin.result | 19 ++++++++++++ mysql-test/t/subselect_extra_no_semijoin.test | 31 ++++++++++++++++++- sql/item_subselect.cc | 2 +- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/subselect_extra_no_semijoin.result b/mysql-test/r/subselect_extra_no_semijoin.result index 79bca388181..200682b4065 100644 --- a/mysql-test/r/subselect_extra_no_semijoin.result +++ b/mysql-test/r/subselect_extra_no_semijoin.result @@ -482,3 +482,22 @@ DROP TABLE t1,t2; set optimizer_switch= @tmp_subselect_extra_derived; set optimizer_switch= @subselect_extra_no_sj_tmp; set @optimizer_switch_for_subselect_extra_test=null; +# +# MDEV-6439: Server crashes in Explain_union::print_explain with explain in slow log, tis620 charset +# +SET NAMES tis620; +set @tmp= @@global.slow_query_log; +SET GLOBAL slow_query_log = 1; +SET long_query_time = 0.000001; +SET log_slow_verbosity = 'explain'; +CREATE TABLE t1 (a VARCHAR(3)) ENGINE=MyISAM; +SELECT * FROM t1 WHERE a >= ANY ( SELECT 'foo'); +a +SELECT * FROM t1 WHERE a >= ANY ( SELECT 'foo' UNION SELECT 'bar' ); +ERROR HY000: Illegal mix of collations (tis620_thai_ci,COERCIBLE) and (latin1_swedish_ci,IMPLICIT) for operation '<=' +create table t2 (b int); +insert into t2 values (1),(2),(3); +SELECT * FROM t1 WHERE a >= ANY ( SELECT 'foo' FROM t2); +ERROR HY000: Illegal mix of collations (tis620_thai_ci,COERCIBLE) and (latin1_swedish_ci,IMPLICIT) for operation '<=' +drop table t1,t2; +SET GLOBAL slow_query_log=@tmp; diff --git a/mysql-test/t/subselect_extra_no_semijoin.test b/mysql-test/t/subselect_extra_no_semijoin.test index 8aba3dde72b..d8809c7f981 100644 --- a/mysql-test/t/subselect_extra_no_semijoin.test +++ b/mysql-test/t/subselect_extra_no_semijoin.test @@ -6,4 +6,33 @@ set @optimizer_switch_for_subselect_extra_test='semijoin=off,firstmatch=off,loo set optimizer_switch= @subselect_extra_no_sj_tmp; -set @optimizer_switch_for_subselect_extra_test=null; \ No newline at end of file +set @optimizer_switch_for_subselect_extra_test=null; + +--echo # +--echo # MDEV-6439: Server crashes in Explain_union::print_explain with explain in slow log, tis620 charset +--echo # + +## Using a separate client connection is easier than restoring state +connect(con1,localhost,root,,); + +SET NAMES tis620; +set @tmp= @@global.slow_query_log; +SET GLOBAL slow_query_log = 1; +SET long_query_time = 0.000001; +SET log_slow_verbosity = 'explain'; + +CREATE TABLE t1 (a VARCHAR(3)) ENGINE=MyISAM; +SELECT * FROM t1 WHERE a >= ANY ( SELECT 'foo'); +--error ER_CANT_AGGREGATE_2COLLATIONS +SELECT * FROM t1 WHERE a >= ANY ( SELECT 'foo' UNION SELECT 'bar' ); + +create table t2 (b int); +insert into t2 values (1),(2),(3); + +--error ER_CANT_AGGREGATE_2COLLATIONS +SELECT * FROM t1 WHERE a >= ANY ( SELECT 'foo' FROM t2); + +drop table t1,t2; +SET GLOBAL slow_query_log=@tmp; +disconnect con1; +connection default; diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index a8dfdff9809..14b2ccd3985 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -575,7 +575,7 @@ bool Item_subselect::is_expensive() */ if (cur_join->optimized && (cur_join->zero_result_cause || !cur_join->tables_list)) - return false; + continue; /* If a subquery is not optimized we cannot estimate its cost. A subquery is From f926c5f4fa34b9db9e0da014e530381209c1af3d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 16 Aug 2018 08:37:54 +0100 Subject: [PATCH 028/111] MDEV-16996 mariabackup --prepare does not use native AIO on Linux by default --- extra/mariabackup/xtrabackup.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index a7d136be820..5673c94fb0e 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1216,7 +1216,7 @@ struct my_option xb_server_options[] = "Use native AIO if supported on this platform.", (G_PTR*) &srv_use_native_aio, (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG, - FALSE, 0, 0, 0, 0, 0}, + TRUE, 0, 0, 0, 0, 0}, {"innodb_page_size", OPT_INNODB_PAGE_SIZE, "The universal page size of the database.", (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0, From 1b49c894291d3d51d6bae2821eee0bf98ee20801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 16 Aug 2018 15:29:40 +0300 Subject: [PATCH 029/111] Re-enable the test mariabackup.unsupported_redo Remove the logic for skipping the test if a log checkpoint occurred, and the logic for tolerating failures. Thanks to MDEV-16791, MLOG_INDEX_LOAD is supposed to always work. --- mysql-test/suite/mariabackup/disabled.def | 1 - .../suite/mariabackup/unsupported_redo.result | 2 +- .../suite/mariabackup/unsupported_redo.test | 35 +------------------ 3 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 mysql-test/suite/mariabackup/disabled.def diff --git a/mysql-test/suite/mariabackup/disabled.def b/mysql-test/suite/mariabackup/disabled.def deleted file mode 100644 index a7be04bfc16..00000000000 --- a/mysql-test/suite/mariabackup/disabled.def +++ /dev/null @@ -1 +0,0 @@ -unsupported_redo : MDEV-16791 allows optimized redo \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/unsupported_redo.result b/mysql-test/suite/mariabackup/unsupported_redo.result index a1f95c099cd..fbad89be0ac 100644 --- a/mysql-test/suite/mariabackup/unsupported_redo.result +++ b/mysql-test/suite/mariabackup/unsupported_redo.result @@ -7,7 +7,7 @@ call mtr.add_suppression("InnoDB: Cannot open datafile for read-only: "); call mtr.add_suppression("Table .* in the InnoDB data dictionary has tablespace id .*, but tablespace with that id or name does not exist"); CREATE TABLE t1(i INT PRIMARY KEY auto_increment, a int) ENGINE INNODB; ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; -# Fails during full backup +# No longer fails during full backup DROP TABLE t1; CREATE TABLE t1(i INT PRIMARY KEY auto_increment, a int) ENGINE INNODB; INSERT INTO t1(a) select 1 union select 2 union select 3; diff --git a/mysql-test/suite/mariabackup/unsupported_redo.test b/mysql-test/suite/mariabackup/unsupported_redo.test index 319ee2c7571..b9456751b9c 100644 --- a/mysql-test/suite/mariabackup/unsupported_redo.test +++ b/mysql-test/suite/mariabackup/unsupported_redo.test @@ -11,33 +11,15 @@ let $basedir=$MYSQLTEST_VARDIR/tmp/backup; let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; CREATE TABLE t1(i INT PRIMARY KEY auto_increment, a int) ENGINE INNODB; ---source ../../suite/innodb/include/no_checkpoint_start.inc ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; -# Below mariabackup operation may complete successfully if checkpoint happens -# after the alter table command. - -echo # Fails during full backup; +echo # No longer fails during full backup; --disable_result_log ---error 0,1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; --enable_result_log DROP TABLE t1; ---let MYSQLD_DATADIR=$basedir/ -perl; -open(OUT, ">$ENV{MYSQLTEST_VARDIR}/log/check.txt") || die; -print OUT ' ---let no_checkpoint_end=1 ---let CLEANUP_IF_CHECKPOINT=rmdir $basedir; ---source ../../suite/innodb/include/no_checkpoint_end.inc ---exit Backup failed to fail despite MLOG_INDEX_LOAD record -' if (-f "$ENV{MYSQLD_DATADIR}/xtrabackup_info"); -close(OUT); -EOF ---source $MYSQLTEST_VARDIR/log/check.txt ---remove_file $MYSQLTEST_VARDIR/log/check.txt rmdir $basedir; CREATE TABLE t1(i INT PRIMARY KEY auto_increment, a int) ENGINE INNODB; @@ -50,29 +32,14 @@ INSERT INTO t1(a) select 1 union select 2 union select 3; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; --enable_result_log ---source ../../suite/innodb/include/no_checkpoint_start.inc ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; --disable_result_log ---error 0,1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir; --enable_result_log DROP TABLE t1; ---let MYSQLD_DATADIR=$incremental_dir/ -perl; -open(OUT, ">$ENV{MYSQLTEST_VARDIR}/log/check.txt") || die; -print OUT ' ---let no_checkpoint_end=1 ---let CLEANUP_IF_CHECKPOINT=rmdir $basedir;rmdir $incremental_dir; ---source ../../suite/innodb/include/no_checkpoint_end.inc ---exit Backup failed to fail despite MLOG_INDEX_LOAD record -' if (-f "$ENV{MYSQLD_DATADIR}/xtrabackup_info"); -close(OUT); -EOF ---source $MYSQLTEST_VARDIR/log/check.txt ---remove_file $MYSQLTEST_VARDIR/log/check.txt rmdir $basedir;rmdir $incremental_dir; CREATE TABLE t1(i INT) ENGINE INNODB; From d6f7fd601637a070ccf922073c0ef9a41a129717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 16 Aug 2018 16:10:18 +0300 Subject: [PATCH 030/111] MDEV-13564: Refuse MLOG_TRUNCATE in mariabackup The MySQL 5.7 TRUNCATE TABLE is inherently incompatible with hot backup, because it is creating and deleting a separate log file, and it is not writing redo log for all changes of the InnoDB data dictionary tables. Refuse to create a corrupted backup if the unsafe form of TRUNCATE was executed. Note: Undo log tablespace truncation cannot be detected easily. Also it is incompatible with backup, for similar reasons. xtrabackup_backup_func(): "Subscribe to" the log events before the first invocation of xtrabackup_copy_logfile(). recv_parse_or_apply_log_rec_body(): If the function pointer log_truncate is set, invoke it to report MLOG_TRUNCATE. --- extra/mariabackup/xtrabackup.cc | 20 +++++++++++++------ .../mariabackup/truncate_during_backup.result | 4 ++++ .../mariabackup/truncate_during_backup.test | 19 ++++++++++++++++++ storage/innobase/include/log0recv.h | 9 +++++++-- storage/innobase/log/log0recv.cc | 13 +++++++++++- 5 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/mariabackup/truncate_during_backup.result create mode 100644 mysql-test/suite/mariabackup/truncate_during_backup.test diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 5673c94fb0e..d9397b001b7 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -358,9 +358,8 @@ struct ddl_tracker_t { /* For DDL operation found in redo log, */ space_id_to_name_t id_to_name; }; -const space_id_t REMOVED_SPACE_ID = ULINT_MAX; -static ddl_tracker_t ddl_tracker; +static ddl_tracker_t ddl_tracker; /* Whether xtrabackup_binlog_info should be created on recovery */ static bool recover_binlog_info; @@ -618,9 +617,8 @@ void backup_file_op(ulint space_id, const byte* flags, /** Callback whenever MLOG_INDEX_LOAD happens. -@param[in] space_id space id to check -@return false */ -void backup_optimized_ddl_op(ulint space_id) +@param[in] space_id space id to check */ +static void backup_optimized_ddl_op(ulint space_id) { // TODO : handle incremental if (xtrabackup_incremental) @@ -631,6 +629,15 @@ void backup_optimized_ddl_op(ulint space_id) pthread_mutex_unlock(&backup_mutex); } +/** Callback whenever MLOG_TRUNCATE happens. */ +static void backup_truncate_fail() +{ + msg("mariabackup: Incompatible TRUNCATE operation detected.%s\n", + opt_lock_ddl_per_table + ? "" + : " Use --lock-ddl-per-table to lock all tables before backup."); +} + /* ======== Date copying thread context ======== */ typedef struct { @@ -4240,12 +4247,13 @@ fail_before_log_copying_thread_start: /* copy log file by current position */ log_copy_scanned_lsn = checkpoint_lsn_start; recv_sys->recovered_lsn = log_copy_scanned_lsn; + log_optimized_ddl_op = backup_optimized_ddl_op; + log_truncate = backup_truncate_fail; if (xtrabackup_copy_logfile()) goto fail_before_log_copying_thread_start; log_copying_stop = os_event_create(0); - log_optimized_ddl_op = backup_optimized_ddl_op; os_thread_create(log_copying_thread, NULL, &log_copying_thread_id); /* FLUSH CHANGED_PAGE_BITMAPS call */ diff --git a/mysql-test/suite/mariabackup/truncate_during_backup.result b/mysql-test/suite/mariabackup/truncate_during_backup.result new file mode 100644 index 00000000000..f9d5c785749 --- /dev/null +++ b/mysql-test/suite/mariabackup/truncate_during_backup.result @@ -0,0 +1,4 @@ +CREATE TABLE t1 ENGINE=InnoDB SELECT 1; +DROP TABLE t1; +SET GLOBAL innodb_log_checkpoint_now=1; +SET GLOBAL innodb_log_checkpoint_now=DEFAULT; diff --git a/mysql-test/suite/mariabackup/truncate_during_backup.test b/mysql-test/suite/mariabackup/truncate_during_backup.test new file mode 100644 index 00000000000..c01a74588e7 --- /dev/null +++ b/mysql-test/suite/mariabackup/truncate_during_backup.test @@ -0,0 +1,19 @@ +--source include/have_debug.inc +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +mkdir $targetdir; + +CREATE TABLE t1 ENGINE=InnoDB SELECT 1; + +--let after_load_tablespaces=TRUNCATE test.t1 + +--disable_result_log +--error 1 +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events; +--enable_result_log + +--let after_load_tablespaces= + +DROP TABLE t1; +SET GLOBAL innodb_log_checkpoint_now=1; +SET GLOBAL innodb_log_checkpoint_now=DEFAULT; +rmdir $targetdir; diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 6716b1cbd0d..28a4327ec4d 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -153,10 +153,15 @@ bool recv_parse_log_recs(lsn_t checkpoint_lsn, store_t store, bool apply); /** Moves the parsing buffer data left to the buffer start. */ void recv_sys_justify_left_parsing_buf(); -/** Report optimized DDL operation (without redo log), corresponding to MLOG_INDEX_LOAD. +/** Report optimized DDL operation (without redo log), +corresponding to MLOG_INDEX_LOAD. @param[in] space_id tablespace identifier */ -extern void(*log_optimized_ddl_op)(ulint space_id); +extern void (*log_optimized_ddl_op)(ulint space_id); + +/** Report backup-unfriendly TRUNCATE operation (with separate log file), +corresponding to MLOG_TRUNCATE. */ +extern void (*log_truncate)(); /** Report an operation to create, delete, or rename a file during backup. @param[in] space_id tablespace identifier diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index ee4d627261a..c623b884628 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -169,11 +169,16 @@ typedef std::map< static recv_spaces_t recv_spaces; -/** Report optimized DDL operation (without redo log), corresponding to MLOG_INDEX_LOAD. +/** Report optimized DDL operation (without redo log), +corresponding to MLOG_INDEX_LOAD. @param[in] space_id tablespace identifier */ void (*log_optimized_ddl_op)(ulint space_id); +/** Report backup-unfriendly TRUNCATE operation (with separate log file), +corresponding to MLOG_TRUNCATE. */ +void (*log_truncate)(); + /** Report an operation to create, delete, or rename a file during backup. @param[in] space_id tablespace identifier @param[in] flags tablespace flags (NULL if not create) @@ -1196,6 +1201,12 @@ recv_parse_or_apply_log_rec_body( } return(ptr + 8); case MLOG_TRUNCATE: + if (log_truncate) { + ut_ad(srv_operation != SRV_OPERATION_NORMAL); + log_truncate(); + recv_sys->found_corrupt_fs = true; + return NULL; + } return(truncate_t::parse_redo_entry(ptr, end_ptr, space_id)); default: From 34c7222c088ded8f1192142db935c000b2ba6b8d Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 16 Aug 2018 17:38:20 +0300 Subject: [PATCH 031/111] Fixed that -DDBUG_ASSERT_AS_PRINTF works again Note that this option was only designed to work on binaries compiled without DBUG. --- sql/field.cc | 8 +++++--- sql/filesort.cc | 4 +++- sql/handler.cc | 4 ++-- sql/sql_select.cc | 2 +- storage/innobase/handler/ha_innodb.cc | 4 ++-- storage/innobase/handler/handler0alter.cc | 12 ++++++------ storage/innobase/trx/trx0undo.cc | 2 +- 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/sql/field.cc b/sql/field.cc index 61bc2daef8d..7eb277e23d8 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7274,7 +7274,9 @@ int Field_string::cmp(const uchar *a_ptr, const uchar *b_ptr) void Field_string::sort_string(uchar *to,uint length) { - IF_DBUG(size_t tmp= ,) +#ifdef DBUG_ASSERT_EXISTS + size_t tmp= +#endif field_charset->coll->strnxfrm(field_charset, to, length, char_length() * @@ -7679,7 +7681,7 @@ void Field_varstring::sort_string(uchar *to,uint length) length-= length_bytes; } -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS size_t rc= #endif field_charset->coll->strnxfrm(field_charset, to, length, @@ -8547,7 +8549,7 @@ void Field_blob::sort_string(uchar *to,uint length) store_bigendian(buf.length(), to + length, packlength); } -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS size_t rc= #endif field_charset->coll->strnxfrm(field_charset, to, length, length, diff --git a/sql/filesort.cc b/sql/filesort.cc index b2c88cda7b3..a4be9e4acfa 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -1010,7 +1010,9 @@ Type_handler_string_result::make_sort_key(uchar *to, Item *item, if (use_strnxfrm(cs)) { - IF_DBUG(size_t tmp_length= ,) +#ifdef DBUG_ASSERT_EXISTS + size_t tmp_length= +#endif cs->coll->strnxfrm(cs, to, sort_field->length, item->max_char_length() * cs->strxfrm_multiply, diff --git a/sql/handler.cc b/sql/handler.cc index 81b305220df..b8a98a0c9f5 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -74,7 +74,7 @@ KEY_CREATE_INFO default_key_create_info= ulong total_ha= 0; /* number of storage engines (from handlertons[]) that support 2pc */ ulong total_ha_2pc= 0; -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS /* Number of non-mandatory 2pc handlertons whose initialization failed to estimate total_ha_2pc value under supposition of the failures @@ -661,7 +661,7 @@ err_deinit: (void) plugin->plugin->deinit(NULL); err: -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS if (hton->prepare && hton->state == SHOW_OPTION_YES) failed_ha_2pc++; #endif diff --git a/sql/sql_select.cc b/sql/sql_select.cc index e7a4289ca22..c3df580f1b7 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -3757,7 +3757,7 @@ bool JOIN::save_explain_data(Explain_query *output, bool can_overwrite, If there is SELECT in this statement with the same number it must be the same SELECT */ - DBUG_ASSERT(select_lex->select_number == UINT_MAX || + DBUG_SLOW_ASSERT(select_lex->select_number == UINT_MAX || select_lex->select_number == INT_MAX || !output || !output->get_select(select_lex->select_number) || diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e2b61b627de..52d4c815dc3 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -16065,7 +16065,7 @@ static int innodb_show_mutex_status( handlerton* -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS hton #endif , @@ -16096,7 +16096,7 @@ static int innodb_show_rwlock_status( handlerton* -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS hton #endif , diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 9c6791a3e28..3f31f253b3f 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -5350,14 +5350,14 @@ new_clustered_failed: = dict_table_get_nth_col(user_table, i)); ut_d(const dict_index_t* index = user_table->indexes.start); - DBUG_ASSERT(col->mtype == old_col->mtype); - DBUG_ASSERT(col->prtype == old_col->prtype); - DBUG_ASSERT(col->mbminlen + DBUG_SLOW_ASSERT(col->mtype == old_col->mtype); + DBUG_SLOW_ASSERT(col->prtype == old_col->prtype); + DBUG_SLOW_ASSERT(col->mbminlen == old_col->mbminlen); - DBUG_ASSERT(col->mbmaxlen + DBUG_SLOW_ASSERT(col->mbmaxlen == old_col->mbmaxlen); - DBUG_ASSERT(col->len >= old_col->len); - DBUG_ASSERT(old_col->is_instant() + DBUG_SLOW_ASSERT(col->len >= old_col->len); + DBUG_SLOW_ASSERT(old_col->is_instant() == (dict_col_get_clust_pos( old_col, index) >= index->n_core_fields)); diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 18755d0457a..63ec73fc2b3 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -1444,7 +1444,7 @@ buf_block_t* trx_undo_assign_low(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, dberr_t* err, mtr_t* mtr) { - const bool is_temp = rseg == trx->rsegs.m_noredo.rseg; + const bool is_temp __attribute__((unused)) = rseg == trx->rsegs.m_noredo.rseg; ut_ad(rseg == trx->rsegs.m_redo.rseg || rseg == trx->rsegs.m_noredo.rseg); From 4ff7f14feffc8cbb685910ecd595095629f7a764 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Fri, 17 Aug 2018 15:54:55 +0200 Subject: [PATCH 032/111] Fixes of the base patch for compatibility with the 10.1 branch --- .../galera/r/galera_sst_rsync_data_dir.result | 26 ------------------- scripts/wsrep_sst_rsync.sh | 2 +- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result b/mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result index ff85a7d6c0f..cec0f21ee22 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result +++ b/mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result @@ -1,7 +1,4 @@ -connection node_1; -connection node_2; Performing State Transfer on a server that has been shut down cleanly and restarted -connection node_1; CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -11,7 +8,6 @@ INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); COMMIT; -connection node_2; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node2_committed_before'); @@ -21,7 +17,6 @@ INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; Shutting down server ... -connection node_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_committed_during'); @@ -36,7 +31,6 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); -connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -44,7 +38,6 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); -connection node_2; Starting server ... SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -54,7 +47,6 @@ INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); COMMIT; -connection node_1; INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); @@ -69,7 +61,6 @@ INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); COMMIT; -connection node_1a_galera_st_shutdown_slave; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -84,7 +75,6 @@ COUNT(*) = 0 1 COMMIT; SET AUTOCOMMIT=ON; -connection node_1; SELECT COUNT(*) = 35 FROM t1; COUNT(*) = 35 1 @@ -96,7 +86,6 @@ COMMIT; SET AUTOCOMMIT=ON; Performing State Transfer on a server that starts from a clean var directory This is accomplished by shutting down node #2 and removing its var directory before restarting it -connection node_1; CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -106,7 +95,6 @@ INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); COMMIT; -connection node_2; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node2_committed_before'); @@ -116,7 +104,6 @@ INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; Shutting down server ... -connection node_1; Cleaning var directory ... SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -132,7 +119,6 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); -connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -140,7 +126,6 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); -connection node_2; Starting server ... SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -150,7 +135,6 @@ INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); COMMIT; -connection node_1; INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); @@ -165,7 +149,6 @@ INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); COMMIT; -connection node_1a_galera_st_clean_slave; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -180,7 +163,6 @@ COUNT(*) = 0 1 COMMIT; SET AUTOCOMMIT=ON; -connection node_1; SELECT COUNT(*) = 35 FROM t1; COUNT(*) = 35 1 @@ -191,7 +173,6 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; Performing State Transfer on a server that has been killed and restarted -connection node_1; CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -201,7 +182,6 @@ INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); COMMIT; -connection node_2; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node2_committed_before'); @@ -211,7 +191,6 @@ INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; Killing server ... -connection node_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_committed_during'); @@ -226,7 +205,6 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); -connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -234,7 +212,6 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); -connection node_2; Performing --wsrep-recover ... Starting server ... Using --wsrep-start-position when starting mysqld ... @@ -246,7 +223,6 @@ INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); COMMIT; -connection node_1; INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); @@ -261,7 +237,6 @@ INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); COMMIT; -connection node_1a_galera_st_kill_slave; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -276,7 +251,6 @@ COUNT(*) = 0 1 COMMIT; SET AUTOCOMMIT=ON; -connection node_1; SELECT COUNT(*) = 35 FROM t1; COUNT(*) = 35 1 diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index f787262085b..77a25ffc530 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -144,7 +144,7 @@ fi # New filter - exclude everything except dirs (schemas) and innodb files FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' -f '+ /wsrep_sst_binlog.tar' -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '- $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*'" + -f '- $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*') SSTKEY=$(parse_cnf sst tkey "") SSTCERT=$(parse_cnf sst tcert "") From 4eac5df3fcebb1adf52e33d9d88dc05bc1e339ce Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 17 Aug 2018 14:27:42 -0700 Subject: [PATCH 033/111] MDEV-16934 Query with very large IN clause lists runs slowly This patch introduces support for the system variable eq_range_index_dive_limit that existed in MySQL starting from 5.6. The variable sets a limit for index dives into equality ranges. Index dives are performed by optimizer to estimate the number of rows in range scans. Index dives usually provide good estimate but they are pretty expensive. To estimate the number of rows in equality ranges statistical data on indexes can be employed. Its usage gives not so good estimates but it's cheap. So if the number of equality dives required by an index scan exceeds the set limit no dives for equality ranges are performed by the optimizer for this index. As the new system variable is introduced in a stable version the default value for it is set to a special value meaning there is no limit for the number of index dives performed by the optimizer. The patch partially uses the MySQL code for WL 5957 'Statistics-based Range optimization for many ranges'. --- mysql-test/r/mysqld--help.result | 6 +++ mysql-test/r/range.result | 41 ++++++++++++++ mysql-test/r/range_mrr_icp.result | 41 ++++++++++++++ .../sys_vars/r/sysvars_server_embedded.result | 14 +++++ .../r/sysvars_server_notembedded.result | 14 +++++ mysql-test/t/range.test | 33 ++++++++++++ sql/multi_range_read.cc | 11 ++++ sql/opt_range.cc | 27 ++++++++++ sql/opt_range.h | 4 +- sql/opt_range_mrr.cc | 54 +++++++++++++------ sql/sql_class.h | 1 + sql/sql_statistics.h | 2 +- sql/sys_vars.cc | 10 ++++ 13 files changed, 239 insertions(+), 19 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 3b7d5e21c00..51ece335908 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -197,6 +197,11 @@ The following specify which files/extra groups are read (specified before remain cache, etc) --enforce-storage-engine=name Force the use of a storage engine for new tables + --eq-range-index-dive-limit=# + The optimizer will use existing index statistics instead + of doing index dives for equality ranges if the number of + equality ranges for the index is larger than or equal to + this number. If set to 0, index dives are always used. --event-scheduler[=name] Enable the event scheduler. Possible values are ON, OFF, and DISABLED (keep the event scheduler completely @@ -1259,6 +1264,7 @@ encrypt-binlog FALSE encrypt-tmp-disk-tables FALSE encrypt-tmp-files FALSE enforce-storage-engine (No default value) +eq-range-index-dive-limit 0 event-scheduler OFF expensive-subquery-limit 100 expire-logs-days 0 diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result index 3a71d08eb38..0c6be5e56f2 100644 --- a/mysql-test/r/range.result +++ b/mysql-test/r/range.result @@ -3002,5 +3002,46 @@ deallocate prepare stmt; set optimizer_switch=@save_optimizer_switch; drop table t1,t2,t3; # +# MDEV-16934: using system variable eq_range_index_dive_limit +# to reduce the number of index dives +# +create table t1 (a int, b varchar(31), index idx(a)); +insert into t1 values +(7,'xxxx'), (1,'yy'), (3,'aaa'), (1,'bbb'), (2,'zz'), +(4,'vvvvv'), (7,'ddd'), (9,'zzzzz'), (1,'cc'), (5,'ffff'); +insert into t1 select a+10, concat(b,'zz') from t1; +insert into t1 select a+15, concat(b,'yy') from t1; +insert into t1 select a+100, concat(b,'xx') from t1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +select cast(count(a)/count(distinct a) as unsigned) as rec_per_key from t1; +rec_per_key +2 +set eq_range_index_dive_limit=0; +explain select * from t1 where a in (8, 15, 31, 1, 9); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 5 NULL 7 Using index condition +select * from t1 where a in (8, 15, 31, 1, 9); +a b +1 yy +1 bbb +1 cc +9 zzzzz +15 ffffzz +set eq_range_index_dive_limit=2; +explain select * from t1 where a in (8, 15, 31, 1, 9); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 5 NULL 10 Using index condition +select * from t1 where a in (8, 15, 31, 1, 9); +a b +1 yy +1 bbb +1 cc +9 zzzzz +15 ffffzz +set eq_range_index_dive_limit=default; +drop table t1; +# # End of 10.2 tests # diff --git a/mysql-test/r/range_mrr_icp.result b/mysql-test/r/range_mrr_icp.result index 799a299e33f..93f414f1604 100644 --- a/mysql-test/r/range_mrr_icp.result +++ b/mysql-test/r/range_mrr_icp.result @@ -3014,6 +3014,47 @@ deallocate prepare stmt; set optimizer_switch=@save_optimizer_switch; drop table t1,t2,t3; # +# MDEV-16934: using system variable eq_range_index_dive_limit +# to reduce the number of index dives +# +create table t1 (a int, b varchar(31), index idx(a)); +insert into t1 values +(7,'xxxx'), (1,'yy'), (3,'aaa'), (1,'bbb'), (2,'zz'), +(4,'vvvvv'), (7,'ddd'), (9,'zzzzz'), (1,'cc'), (5,'ffff'); +insert into t1 select a+10, concat(b,'zz') from t1; +insert into t1 select a+15, concat(b,'yy') from t1; +insert into t1 select a+100, concat(b,'xx') from t1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +select cast(count(a)/count(distinct a) as unsigned) as rec_per_key from t1; +rec_per_key +2 +set eq_range_index_dive_limit=0; +explain select * from t1 where a in (8, 15, 31, 1, 9); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 5 NULL 7 Using index condition; Rowid-ordered scan +select * from t1 where a in (8, 15, 31, 1, 9); +a b +1 yy +1 bbb +9 zzzzz +1 cc +15 ffffzz +set eq_range_index_dive_limit=2; +explain select * from t1 where a in (8, 15, 31, 1, 9); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 5 NULL 10 Using index condition; Rowid-ordered scan +select * from t1 where a in (8, 15, 31, 1, 9); +a b +1 yy +1 bbb +9 zzzzz +1 cc +15 ffffzz +set eq_range_index_dive_limit=default; +drop table t1; +# # End of 10.2 tests # set optimizer_switch=@mrr_icp_extra_tmp; diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index a7a7547d73b..31a6b597edc 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -779,6 +779,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT NULL +VARIABLE_NAME EQ_RANGE_INDEX_DIVE_LIMIT +SESSION_VALUE 0 +GLOBAL_VALUE 0 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 0 +VARIABLE_SCOPE SESSION +VARIABLE_TYPE INT UNSIGNED +VARIABLE_COMMENT The optimizer will use existing index statistics instead of doing index dives for equality ranges if the number of equality ranges for the index is larger than or equal to this number. If set to 0, index dives are always used. +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 4294967295 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME ERROR_COUNT SESSION_VALUE 0 GLOBAL_VALUE NULL diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index a8f20769e30..133b1fc8969 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -793,6 +793,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT NULL +VARIABLE_NAME EQ_RANGE_INDEX_DIVE_LIMIT +SESSION_VALUE 0 +GLOBAL_VALUE 0 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 0 +VARIABLE_SCOPE SESSION +VARIABLE_TYPE INT UNSIGNED +VARIABLE_COMMENT The optimizer will use existing index statistics instead of doing index dives for equality ranges if the number of equality ranges for the index is larger than or equal to this number. If set to 0, index dives are always used. +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 4294967295 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME ERROR_COUNT SESSION_VALUE 0 GLOBAL_VALUE NULL diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test index ab951809b7a..ed68cfb9583 100644 --- a/mysql-test/t/range.test +++ b/mysql-test/t/range.test @@ -2045,6 +2045,39 @@ set optimizer_switch=@save_optimizer_switch; drop table t1,t2,t3; +--echo # +--echo # MDEV-16934: using system variable eq_range_index_dive_limit +--echo # to reduce the number of index dives +--echo # + +create table t1 (a int, b varchar(31), index idx(a)); + +insert into t1 values + (7,'xxxx'), (1,'yy'), (3,'aaa'), (1,'bbb'), (2,'zz'), + (4,'vvvvv'), (7,'ddd'), (9,'zzzzz'), (1,'cc'), (5,'ffff'); +insert into t1 select a+10, concat(b,'zz') from t1; +insert into t1 select a+15, concat(b,'yy') from t1; +insert into t1 select a+100, concat(b,'xx') from t1; + +analyze table t1; + +select cast(count(a)/count(distinct a) as unsigned) as rec_per_key from t1; + +let $q= +select * from t1 where a in (8, 15, 31, 1, 9); + +set eq_range_index_dive_limit=0; +eval explain $q; +eval $q; + +set eq_range_index_dive_limit=2; +eval explain $q; +eval $q; + +set eq_range_index_dive_limit=default; + +drop table t1; + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc index 50918d8dcf2..94a96c27a0f 100644 --- a/sql/multi_range_read.cc +++ b/sql/multi_range_read.cc @@ -17,6 +17,7 @@ #include #include "sql_select.h" #include "key.h" +#include "sql_statistics.h" /**************************************************************************** * Default MRR implementation (MRR to non-MRR converter) @@ -67,6 +68,9 @@ handler::multi_range_read_info_const(uint keyno, RANGE_SEQ_IF *seq, /* Default MRR implementation doesn't need buffer */ *bufsz= 0; + bool use_statistics_for_eq_range= eq_ranges_exceeds_limit(seq, + seq_init_param); + seq_it= seq->init(seq_init_param, n_ranges, *flags); while (!seq->next(seq_it, &range)) { @@ -87,8 +91,15 @@ handler::multi_range_read_info_const(uint keyno, RANGE_SEQ_IF *seq, min_endp= range.start_key.length? &range.start_key : NULL; max_endp= range.end_key.length? &range.end_key : NULL; } + int keyparts_used= my_count_bits(range.start_key.keypart_map); if ((range.range_flag & UNIQUE_RANGE) && !(range.range_flag & NULL_RANGE)) rows= 1; /* there can be at most one row */ + else if (use_statistics_for_eq_range && + !(range.range_flag & NULL_RANGE) && + (range.range_flag & EQ_RANGE) && + table->key_info[keyno].actual_rec_per_key(keyparts_used - 1) > 0.5) + rows= + (ha_rows) table->key_info[keyno].actual_rec_per_key(keyparts_used - 1); else { if (HA_POS_ERROR == (rows= this->records_in_range(keyno, min_endp, diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 0d723094445..47e07016782 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -14617,6 +14617,33 @@ void QUICK_GROUP_MIN_MAX_SELECT::add_keys_and_lengths(String *key_names, } +/* Check whether the number for equality ranges exceeds the set threshold */ + +bool eq_ranges_exceeds_limit(RANGE_SEQ_IF *seq, void *seq_init_param) +{ + KEY_MULTI_RANGE range; + range_seq_t seq_it; + uint count = 0; + PARAM *param= ((SEL_ARG_RANGE_SEQ*) seq_init_param)->param; + uint limit= param->thd->variables.eq_range_index_dive_limit; + + if (limit == 0) + { + /* 'Statistics instead of index dives' feature is turned off */ + return false; + } + seq_it= seq->init(seq_init_param, 0, 0); + while (!seq->next(seq_it, &range)) + { + if ((range.range_flag & EQ_RANGE) && !(range.range_flag & NULL_RANGE)) + { + if (++count >= limit) + return true; + } + } + return false; +} + #ifndef DBUG_OFF static void print_sel_tree(PARAM *param, SEL_TREE *tree, key_map *tree_map, diff --git a/sql/opt_range.h b/sql/opt_range.h index c1f7079ce7f..6698c987e78 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -242,7 +242,7 @@ public: Number of children of this element in the RB-tree, plus 1 for this element itself. */ - uint16 elements; + uint32 elements; /* Valid only for elements which are RB-tree roots: Number of times this RB-tree is referred to (it is referred by SEL_ARG::next_key_part or by @@ -1664,6 +1664,8 @@ SQL_SELECT *make_select(TABLE *head, table_map const_tables, bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item **cond); +bool eq_ranges_exceeds_limit(RANGE_SEQ_IF *seq, void *seq_init_param); + #ifdef WITH_PARTITION_STORAGE_ENGINE bool prune_partitions(THD *thd, TABLE *table, Item *pprune_cond); #endif diff --git a/sql/opt_range_mrr.cc b/sql/opt_range_mrr.cc index ace6208fd77..c7e4496f2c1 100644 --- a/sql/opt_range_mrr.cc +++ b/sql/opt_range_mrr.cc @@ -72,6 +72,7 @@ typedef struct st_sel_arg_range_seq range_seq_t sel_arg_range_seq_init(void *init_param, uint n_ranges, uint flags) { SEL_ARG_RANGE_SEQ *seq= (SEL_ARG_RANGE_SEQ*)init_param; + seq->param->range_count=0; seq->at_start= TRUE; seq->stack[0].key_tree= NULL; seq->stack[0].min_key= seq->param->min_key; @@ -272,25 +273,44 @@ walk_up_n_right: key_info= NULL; else key_info= &seq->param->table->key_info[seq->real_keyno]; - + /* - Conditions below: - (1) - range analysis is used for estimating condition selectivity - (2) - This is a unique key, and we have conditions for all its - user-defined key parts. - (3) - The table uses extended keys, this key covers all components, - and we have conditions for all key parts. + This is an equality range (keypart_0=X and ... and keypart_n=Z) if + (1) - There are no flags indicating open range (e.g., + "keypart_x > y") or GIS. + (2) - The lower bound and the upper bound of the range has the + same value (min_key == max_key). */ - if (!(cur->min_key_flag & ~NULL_RANGE) && !cur->max_key_flag && - (!key_info || // (1) - ((uint)key_tree->part+1 == key_info->user_defined_key_parts && // (2) - key_info->flags & HA_NOSAME) || // (2) - ((key_info->flags & HA_EXT_NOSAME) && // (3) - (uint)key_tree->part+1 == key_info->ext_key_parts) // (3) - ) && - range->start_key.length == range->end_key.length && - !memcmp(seq->param->min_key,seq->param->max_key,range->start_key.length)) - range->range_flag= UNIQUE_RANGE | (cur->min_key_flag & NULL_RANGE); + const uint is_open_range = + (NO_MIN_RANGE | NO_MAX_RANGE | NEAR_MIN | NEAR_MAX | GEOM_FLAG); + const bool is_eq_range_pred = + !(cur->min_key_flag & is_open_range) && // (1) + !(cur->max_key_flag & is_open_range) && // (1) + range->start_key.length == range->end_key.length && // (2) + !memcmp(seq->param->min_key, seq->param->max_key, // (2) + range->start_key.length); + + if (is_eq_range_pred) + { + range->range_flag = EQ_RANGE; + + /* + Conditions below: + (1) - Range analysis is used for estimating condition selectivity + (2) - This is a unique key, and we have conditions for all its + user-defined key parts. + (3) - The table uses extended keys, this key covers all components, + and we have conditions for all key parts. + */ + if ( + !key_info || // (1) + ((uint)key_tree->part+1 == key_info->user_defined_key_parts && // (2) + key_info->flags & HA_NOSAME) || // (2) + ((key_info->flags & HA_EXT_NOSAME) && // (3) + (uint)key_tree->part+1 == key_info->ext_key_parts) // (3) + ) + range->range_flag |= UNIQUE_RANGE | (cur->min_key_flag & NULL_RANGE); + } if (seq->param->is_ror_scan) { diff --git a/sql/sql_class.h b/sql/sql_class.h index cb7523af615..4888b38bc30 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -569,6 +569,7 @@ typedef struct system_variables ha_rows max_join_size; ha_rows expensive_subquery_limit; ulong auto_increment_increment, auto_increment_offset; + uint eq_range_index_dive_limit; ulong lock_wait_timeout; ulong join_cache_level; ulong max_allowed_packet; diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 0611c021e88..28d5cd46760 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -21,7 +21,7 @@ enum enum_use_stat_tables_mode { NEVER, COMPLEMENTARY, - PEFERABLY, + PREFERABLY, } Use_stat_tables_mode; typedef diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 841e57fa3e2..391772d1191 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -2584,6 +2584,16 @@ static Sys_var_ulong Sys_div_precincrement( SESSION_VAR(div_precincrement), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, DECIMAL_MAX_SCALE), DEFAULT(4), BLOCK_SIZE(1)); +static Sys_var_uint Sys_eq_range_index_dive_limit( + "eq_range_index_dive_limit", + "The optimizer will use existing index statistics instead of " + "doing index dives for equality ranges if the number of equality " + "ranges for the index is larger than or equal to this number. " + "If set to 0, index dives are always used.", + SESSION_VAR(eq_range_index_dive_limit), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0, UINT_MAX32), DEFAULT(0), + BLOCK_SIZE(1)); + static Sys_var_ulong Sys_range_alloc_block_size( "range_alloc_block_size", "Allocation block size for storing ranges during optimization", From 862a97749d5a36218ba4d55c26eef30cd7b2e3cb Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 17 Aug 2018 19:29:01 -0700 Subject: [PATCH 034/111] =?UTF-8?q?MDEV-17011=20=E2=80=9Ccondition=5Fpushd?= =?UTF-8?q?own=5Ffor=5Fderived=E2=80=9D=20optimization=20not=20used=20when?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20using=20INSERT=20INTO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch allows condition pushdown into a materialized derived / view when this table is used in INSERT SELECT, multi-table UPDATE and multi-table DELETE. --- mysql-test/r/derived_cond_pushdown.result | 143 ++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 44 +++++++ sql/sql_select.cc | 3 +- 3 files changed, 188 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 22b81ef48a5..839ae587839 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -9874,3 +9874,146 @@ SELECT a FROM (SELECT "aa" a) t WHERE f1(t.a, (SELECT MAX('aa') FROM DUAL LIMIT a aa DROP FUNCTION f1; +# +# MDEV-17011: condition pushdown into materialized derived used +# in INSERT SELECT, multi-table UPDATE and DELETE +# +CREATE TABLE t1 (a int ,b int) ENGINE=MyISAM; +INSERT INTO t1 VALUES +(1, 1), (1, 2), (2, 1), (2, 2), (3,1), (3,3), (4,2); +CREATE TABLE t2 (a int) ENGINE MYISAM; +INSERT INTO t2 VALUES +(3), (7), (1), (4), (1); +CREATE TABLE t3 (a int, b int) ENGINE MYISAM; +EXPLAIN FORMAT=JSON INSERT INTO t3 +SELECT * FROM (SELECT a, count(*) as c FROM t1 GROUP BY a) t WHERE a<=2; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 7, + "filtered": 100, + "attached_condition": "t.a <= 2", + "materialized": { + "query_block": { + "select_id": 2, + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 7, + "filtered": 100, + "attached_condition": "t1.a <= 2" + } + } + } + } + } + } + } +} +INSERT INTO t3 +SELECT * FROM (SELECT a, count(*) as c FROM t1 GROUP BY a) t WHERE a<=2; +SELECT * FROM t3; +a b +1 2 +2 2 +EXPLAIN FORMAT=JSON UPDATE t2, (SELECT a, count(*) as c FROM t1 GROUP BY a) t SET t2.a=t.c+10 +WHERE t2.a= t.c and t.a>=3; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 5, + "filtered": 100, + "attached_condition": "t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "8", + "used_key_parts": ["c"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "t2.a = t.c and t.a >= 3", + "materialized": { + "query_block": { + "select_id": 2, + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 7, + "filtered": 100, + "attached_condition": "t1.a >= 3" + } + } + } + } + } + } + } +} +UPDATE t2, (SELECT a, count(*) as c FROM t1 GROUP BY a) t SET t2.a=t.c+10 +WHERE t2.a= t.c and t.a>=3; +SELECT * FROM t2; +a +3 +7 +11 +4 +11 +EXPLAIN FORMAT=JSON DELETE t2 FROM t2, (SELECT a, count(*) as c FROM t1 GROUP BY a) t +WHERE t2.a= t.c+9 and t.a=2; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 5, + "filtered": 100 + }, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 7, + "filtered": 100, + "attached_condition": "t.a = 2 and t2.a = t.c + 9", + "materialized": { + "query_block": { + "select_id": 2, + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 7, + "filtered": 100, + "attached_condition": "t1.a = 2" + } + } + } + } + } +} +DELETE t2 FROM t2, (SELECT a, count(*) as c FROM t1 GROUP BY a) t +WHERE t2.a= t.c+9 and t.a=2; +SELECT * FROM t2; +a +3 +7 +4 +DROP TABLE t1,t2,t3; diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index d9d767f53ed..f85b301de35 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -1920,3 +1920,47 @@ END;$$ DELIMITER ;$$ SELECT a FROM (SELECT "aa" a) t WHERE f1(t.a, (SELECT MAX('aa') FROM DUAL LIMIT 1)); DROP FUNCTION f1; + +--echo # +--echo # MDEV-17011: condition pushdown into materialized derived used +--echo # in INSERT SELECT, multi-table UPDATE and DELETE +--echo # + +CREATE TABLE t1 (a int ,b int) ENGINE=MyISAM; +INSERT INTO t1 VALUES + (1, 1), (1, 2), (2, 1), (2, 2), (3,1), (3,3), (4,2); + +CREATE TABLE t2 (a int) ENGINE MYISAM; +INSERT INTO t2 VALUES + (3), (7), (1), (4), (1); + +CREATE TABLE t3 (a int, b int) ENGINE MYISAM; + +let $q1= +INSERT INTO t3 +SELECT * FROM (SELECT a, count(*) as c FROM t1 GROUP BY a) t WHERE a<=2; + +eval EXPLAIN FORMAT=JSON $q1; +eval $q1; + +SELECT * FROM t3; + +let $q2= +UPDATE t2, (SELECT a, count(*) as c FROM t1 GROUP BY a) t SET t2.a=t.c+10 + WHERE t2.a= t.c and t.a>=3; + +eval EXPLAIN FORMAT=JSON $q2; +eval $q2; + +SELECT * FROM t2; + +let $q3= +DELETE t2 FROM t2, (SELECT a, count(*) as c FROM t1 GROUP BY a) t + WHERE t2.a= t.c+9 and t.a=2; + +eval EXPLAIN FORMAT=JSON $q3; +eval $q3; + +SELECT * FROM t2; + +DROP TABLE t1,t2,t3; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 643ddfeba39..b94ca496b95 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1375,8 +1375,7 @@ JOIN::optimize_inner() DBUG_RETURN(1); } - if (thd->lex->sql_command == SQLCOM_SELECT && - optimizer_flag(thd, OPTIMIZER_SWITCH_COND_PUSHDOWN_FOR_DERIVED)) + if (optimizer_flag(thd, OPTIMIZER_SWITCH_COND_PUSHDOWN_FOR_DERIVED)) { TABLE_LIST *tbl; List_iterator_fast li(select_lex->leaf_tables); From 0de3c423cc51d362c3a9b4dc8fa90d549ce0754e Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Mon, 20 Aug 2018 17:42:49 +0300 Subject: [PATCH 035/111] MDEV-16765: Missing rows with pushdown condition defined with CASE using Item_cond The bug appears because of the wrong pushdown into the WHERE clause of the materialized derived table/view work. For the excl_dep_on_grouping_fields() method that checks if the condition can be pushed into the WHERE clause the case when Item_cond is used is missing. For Item_cond elements this method always returns positive result (that condition can be pushed). So this condition is pushed even if is shouldn't be pushed. To fix it new Item_cond::excl_dep_on_grouping_fields() method is added. --- mysql-test/r/derived_cond_pushdown.result | 203 ++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 58 +++++++ sql/item_cmpfunc.cc | 13 ++ sql/item_cmpfunc.h | 1 + 4 files changed, 275 insertions(+) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 839ae587839..fd930d13547 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -10017,3 +10017,206 @@ a 7 4 DROP TABLE t1,t2,t3; +# +# MDEV-16765: pushdown condition with the CASE structure +# defined with Item_cond item +# +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 VALUES (1,2), (3,4), (2,3); +SELECT * +FROM +( +SELECT CASE WHEN ((tab2.max_a=1) OR (tab2.max_a=2)) +THEN 1 ELSE 0 END AS max_a,b +FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +max_a b +1 2 +1 3 +EXPLAIN FORMAT=JSON SELECT * +FROM +( +SELECT CASE WHEN ((tab2.max_a=1) OR (tab2.max_a=2)) +THEN 1 ELSE 0 END AS max_a,b +FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "case when (tab2.max_a = 1 or tab2.max_a = 2) then 1 else 0 end = 1", + "materialized": { + "query_block": { + "select_id": 3, + "having_condition": "case when (max_a = 1 or max_a = 2) then 1 else 0 end = 1", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + } + } +} +SELECT * +FROM +( +SELECT CASE WHEN ((tab2.max_a=1) OR ((tab2.max_a>2) AND (tab2.max_a<4))) +THEN 1 ELSE 0 END AS max_a,b +FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +max_a b +1 2 +1 4 +EXPLAIN FORMAT=JSON SELECT * +FROM +( +SELECT CASE WHEN ((tab2.max_a=1) OR ((tab2.max_a>2) AND (tab2.max_a<4))) +THEN 1 ELSE 0 END AS max_a,b +FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "case when (tab2.max_a = 1 or tab2.max_a > 2 and tab2.max_a < 4) then 1 else 0 end = 1", + "materialized": { + "query_block": { + "select_id": 3, + "having_condition": "case when (max_a = 1 or max_a > 2 and max_a < 4) then 1 else 0 end = 1", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + } + } +} +SELECT * +FROM +( +SELECT CASE WHEN ((tab2.max_a>1) AND ((tab2.max_a=2) OR (tab2.max_a>2))) +THEN 1 ELSE 0 END AS max_a,b +FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +max_a b +1 3 +1 4 +EXPLAIN FORMAT=JSON SELECT * +FROM +( +SELECT CASE WHEN ((tab2.max_a>1) AND ((tab2.max_a=2) OR (tab2.max_a>2))) +THEN 1 ELSE 0 END AS max_a,b +FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "case when (tab2.max_a > 1 and (tab2.max_a = 2 or tab2.max_a > 2)) then 1 else 0 end = 1", + "materialized": { + "query_block": { + "select_id": 3, + "having_condition": "case when (max_a > 1 and (max_a = 2 or max_a > 2)) then 1 else 0 end = 1", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + } + } +} +SELECT * +FROM +( +SELECT CASE WHEN ((tab2.b=2) OR (tab2.b=4)) +THEN 1 ELSE 0 END AS max_a,b +FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +max_a b +1 2 +1 4 +EXPLAIN FORMAT=JSON SELECT * +FROM +( +SELECT CASE WHEN ((tab2.b=2) OR (tab2.b=4)) +THEN 1 ELSE 0 END AS max_a,b +FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "case when (tab2.b = 2 or tab2.b = 4) then 1 else 0 end = 1", + "materialized": { + "query_block": { + "select_id": 3, + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "case when (t1.b = 2 or t1.b = 4) then 1 else 0 end = 1" + } + } + } + } + } + } + } +} +DROP TABLE t1; diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index f85b301de35..b8e74a2d5aa 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -1964,3 +1964,61 @@ eval $q3; SELECT * FROM t2; DROP TABLE t1,t2,t3; + +--echo # +--echo # MDEV-16765: pushdown condition with the CASE structure +--echo # defined with Item_cond item +--echo # + +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 VALUES (1,2), (3,4), (2,3); + +LET $query= +SELECT * +FROM +( + SELECT CASE WHEN ((tab2.max_a=1) OR (tab2.max_a=2)) + THEN 1 ELSE 0 END AS max_a,b + FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +LET $query= +SELECT * +FROM +( + SELECT CASE WHEN ((tab2.max_a=1) OR ((tab2.max_a>2) AND (tab2.max_a<4))) + THEN 1 ELSE 0 END AS max_a,b + FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +LET $query= +SELECT * +FROM +( + SELECT CASE WHEN ((tab2.max_a>1) AND ((tab2.max_a=2) OR (tab2.max_a>2))) + THEN 1 ELSE 0 END AS max_a,b + FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +LET $query= +SELECT * +FROM +( + SELECT CASE WHEN ((tab2.b=2) OR (tab2.b=4)) + THEN 1 ELSE 0 END AS max_a,b + FROM (SELECT MAX(a) as max_a,b FROM t1 GROUP BY t1.b) AS tab2 +) AS tab1 +WHERE (tab1.max_a=1); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +DROP TABLE t1; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index f176a0a8193..6834820c0b5 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4970,6 +4970,19 @@ Item *Item_cond::build_clone(THD *thd, MEM_ROOT *mem_root) } +bool Item_cond::excl_dep_on_grouping_fields(st_select_lex *sel) +{ + List_iterator_fast li(list); + Item *item; + while ((item= li++)) + { + if (!item->excl_dep_on_grouping_fields(sel)) + return false; + } + return true; +} + + void Item_cond_and::mark_as_condition_AND_part(TABLE_LIST *embedding) { List_iterator li(list); diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 41a51ee8d12..59ac5f56fe1 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -2224,6 +2224,7 @@ public: Item_transformer transformer, uchar *arg_t); bool eval_not_null_tables(void *opt_arg); Item *build_clone(THD *thd, MEM_ROOT *mem_root); + bool excl_dep_on_grouping_fields(st_select_lex *sel); }; template class LI, class T> class Item_equal_iterator; From 0dadb96e169d3fa94b5cbcadd97b3cc8e3c0e138 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 6 Jul 2018 15:34:29 +1000 Subject: [PATCH 036/111] travis: llvm repo gpg key missing - use travis defination --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6142e76e1e0..95e0796099b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -111,8 +111,8 @@ addons: apt: sources: - ubuntu-toolchain-r-test - - sourceline: 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-5.0 main' - - sourceline: 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-6.0 main' + - llvm-toolchain-trusty-5.0 + - llvm-toolchain-trusty-6.0 packages: # make sure these include all compilers and all build dependencies (see list above) - gcc-5 - g++-5 From cccdb176a6eb506426ba63724bf807cc5dfbb844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 21 Aug 2018 08:54:21 +0300 Subject: [PATCH 037/111] MDEV-16862 build failure for WITH_INNODB_AHI=0 Fix the build, which was broken by MDEV-16515. --- storage/innobase/row/row0import.cc | 3 ++- storage/innobase/row/row0mysql.cc | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 117a2e573de..371b9aa556f 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3928,7 +3928,7 @@ row_import_for_mysql( DBUG_EXECUTE_IF("ib_import_reset_space_and_lsn_failure", err = DB_TOO_MANY_CONCURRENT_TRXS;); - +#ifdef BTR_CUR_HASH_ADAPT /* On DISCARD TABLESPACE, we did not drop any adaptive hash index entries. If we replaced the discarded tablespace with a smaller one here, there could still be some adaptive hash @@ -3945,6 +3945,7 @@ row_import_for_mysql( break; } } +#endif /* BTR_CUR_HASH_ADAPT */ if (err != DB_SUCCESS) { char table_name[MAX_FULL_NAME_LEN + 1]; diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 5ba8d038671..e1fe6e15067 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -3433,6 +3433,7 @@ row_drop_table_for_mysql( if (!dict_table_is_temporary(table)) { if (table->space != TRX_SYS_SPACE) { +#ifdef BTR_CUR_HASH_ADAPT /* On DISCARD TABLESPACE, we would not drop the adaptive hash index entries. If the tablespace is missing here, delete-marking the record in SYS_INDEXES @@ -3454,6 +3455,7 @@ row_drop_table_for_mysql( goto funct_exit; } } +#endif /* BTR_CUR_HASH_ADAPT */ /* Delete the link file if used. */ if (DICT_TF_HAS_DATA_DIR(table->flags)) { From 45dbd47026362dc18a5d1717d33191a0e15db61f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 21 Aug 2018 11:56:50 +0300 Subject: [PATCH 038/111] MDEV-17003 service_manager_extend_timeout() being called too often buf_dump(): Only generate the output when shutdown is in progress. log_write_up_to(): Only generate the output before actually writing to the redo log files. srv_purge_should_exit(): Rate-limit the output, and instead of displaying the work done, indicate the work that remains to be done until the completion of the slow shutdown. --- storage/innobase/buf/buf0dump.cc | 4 ++-- storage/innobase/log/log0log.cc | 13 +++++++------ storage/innobase/srv/srv0srv.cc | 14 +++++++++++--- storage/xtradb/buf/buf0dump.cc | 4 ++-- storage/xtradb/log/log0log.cc | 13 +++++++------ storage/xtradb/srv/srv0srv.cc | 14 +++++++++++--- 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 1df75386872..74d71fd97e2 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -334,7 +334,7 @@ buf_dump( i + 1, srv_buf_pool_instances, j + 1, n_pages); } - if ( (j % 1024) == 0) { + if (SHUTTING_DOWN() && !(j % 1024)) { service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, "Dumping buffer pool " ULINTPF "/" ULINTPF ", " diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 6546459354b..cd00394cded 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1443,12 +1443,6 @@ log_write_up_to( return; } - if (srv_shutdown_state != SRV_SHUTDOWN_NONE) { - service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, - "log write up to: " LSN_PF, - lsn); - } - loop: ut_ad(++loop_count < 100); @@ -1561,6 +1555,13 @@ loop: log_sys->buf_free += OS_FILE_LOG_BLOCK_SIZE; log_sys->write_end_offset = log_sys->buf_free; + if (UNIV_UNLIKELY(srv_shutdown_state != SRV_SHUTDOWN_NONE)) { + service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, + "InnoDB log write: " + LSN_PF "," LSN_PF, + log_sys->write_lsn, lsn); + } + group = UT_LIST_GET_FIRST(log_sys->log_groups); /* Do the write to the log files */ diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 6e8ed7893d6..87f9064c14e 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -2542,9 +2542,17 @@ srv_purge_should_exit(ulint n_purged) } /* Slow shutdown was requested. */ if (n_purged) { - service_manager_extend_timeout( - INNODB_EXTEND_TIMEOUT_INTERVAL, - "InnoDB " ULINTPF " pages purged", n_purged); +#if defined HAVE_SYSTEMD && !defined EMBEDDED_LIBRARY + static ib_time_t progress_time; + ib_time_t time = ut_time(); + if (time - progress_time >= 15) { + progress_time = time; + service_manager_extend_timeout( + INNODB_EXTEND_TIMEOUT_INTERVAL, + "InnoDB: to purge " ULINTPF " transactions", + trx_sys->rseg_history_len); + } +#endif /* The previous round still did some work. */ return(false); } diff --git a/storage/xtradb/buf/buf0dump.cc b/storage/xtradb/buf/buf0dump.cc index 90358d34b04..e9168d9f5d5 100644 --- a/storage/xtradb/buf/buf0dump.cc +++ b/storage/xtradb/buf/buf0dump.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -334,7 +334,7 @@ buf_dump( i + 1, srv_buf_pool_instances, j + 1, n_pages); } - if ( (j % 1024) == 0) { + if (SHUTTING_DOWN() && !(j % 1024)) { service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, "Dumping buffer pool " ULINTPF "/" ULINTPF ", " diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 1420f5a3a12..19b4ac3732a 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1561,12 +1561,6 @@ log_write_up_to( return; } - if (srv_shutdown_state != SRV_SHUTDOWN_NONE) { - service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, - "log write up to: " LSN_PF, - lsn); - } - loop: ut_ad(++loop_count < 100); @@ -1679,6 +1673,13 @@ loop: log_sys->buf_free += OS_FILE_LOG_BLOCK_SIZE; log_sys->write_end_offset = log_sys->buf_free; + if (UNIV_UNLIKELY(srv_shutdown_state != SRV_SHUTDOWN_NONE)) { + service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, + "InnoDB log write: " + LSN_PF "," LSN_PF, + log_sys->write_lsn, lsn); + } + group = UT_LIST_GET_FIRST(log_sys->log_groups); /* Do the write to the log files */ diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 31e87095934..abdd31333f4 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -3236,9 +3236,17 @@ srv_purge_should_exit(ulint n_purged) } /* Slow shutdown was requested. */ if (n_purged) { - service_manager_extend_timeout( - INNODB_EXTEND_TIMEOUT_INTERVAL, - "InnoDB " ULINTPF " pages purged", n_purged); +#if defined HAVE_SYSTEMD && !defined EMBEDDED_LIBRARY + static ib_time_t progress_time; + ib_time_t time = ut_time(); + if (time - progress_time >= 15) { + progress_time = time; + service_manager_extend_timeout( + INNODB_EXTEND_TIMEOUT_INTERVAL, + "InnoDB: to purge " ULINTPF " transactions", + trx_sys->rseg_history_len); + } +#endif /* The previous round still did some work. */ return(false); } From dc7c080369472f6f33344299d2e3d01619edf885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 21 Aug 2018 12:01:44 +0300 Subject: [PATCH 039/111] MDEV-17026 Assertion srv_undo_sources || ... failed on slow shutdown trx_purge_add_update_undo_to_history(): Relax the too strict assertion by removing the condition on srv_fast_shutdown (innodb_fast_shutdown). Rollback is allowed during any form of shutdown. --- storage/innobase/trx/trx0purge.cc | 18 ++++++++++-------- storage/xtradb/trx/trx0purge.cc | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 9e5e90128cb..181fde958ff 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -243,18 +243,20 @@ trx_purge_add_update_undo_to_history( hist_size + undo->size, MLOG_4BYTES, mtr); } - /* Before any transaction-generating background threads or the + /* After the purge thread has been given permission to exit, + we may roll back transactions (trx->undo_no==0) + in THD::cleanup() invoked from unlink_thd() in fast shutdown, + or in trx_rollback_resurrected() in slow shutdown. + + Before any transaction-generating background threads or the purge have been started, recv_recovery_rollback_active() can start transactions in row_merge_drop_temp_indexes() and - fts_drop_orphaned_tables(), and roll back recovered transactions. - After the purge thread has been given permission to exit, - in fast shutdown, we may roll back transactions (trx->undo_no==0) - in THD::cleanup() invoked from unlink_thd(). */ + fts_drop_orphaned_tables(), and roll back recovered transactions. */ ut_ad(srv_undo_sources + || trx->undo_no == 0 || ((srv_startup_is_before_trx_rollback_phase || trx_rollback_or_clean_is_active) - && purge_sys->state == PURGE_STATE_INIT) - || (trx->undo_no == 0 && srv_fast_shutdown)); + && purge_sys->state == PURGE_STATE_INIT)); /* Add the log as the first in the history list */ flst_add_first(rseg_header + TRX_RSEG_HISTORY, diff --git a/storage/xtradb/trx/trx0purge.cc b/storage/xtradb/trx/trx0purge.cc index cbf783628f9..893dc8f398c 100644 --- a/storage/xtradb/trx/trx0purge.cc +++ b/storage/xtradb/trx/trx0purge.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -247,18 +247,20 @@ trx_purge_add_update_undo_to_history( hist_size + undo->size, MLOG_4BYTES, mtr); } - /* Before any transaction-generating background threads or the + /* After the purge thread has been given permission to exit, + we may roll back transactions (trx->undo_no==0) + in THD::cleanup() invoked from unlink_thd() in fast shutdown, + or in trx_rollback_resurrected() in slow shutdown. + + Before any transaction-generating background threads or the purge have been started, recv_recovery_rollback_active() can start transactions in row_merge_drop_temp_indexes() and - fts_drop_orphaned_tables(), and roll back recovered transactions. - After the purge thread has been given permission to exit, - in fast shutdown, we may roll back transactions (trx->undo_no==0) - in THD::cleanup() invoked from unlink_thd(). */ + fts_drop_orphaned_tables(), and roll back recovered transactions. */ ut_ad(srv_undo_sources + || trx->undo_no == 0 || ((srv_startup_is_before_trx_rollback_phase || trx_rollback_or_clean_is_active) - && purge_sys->state == PURGE_STATE_INIT) - || (trx->undo_no == 0 && srv_fast_shutdown)); + && purge_sys->state == PURGE_STATE_INIT)); /* Add the log as the first in the history list */ flst_add_first(rseg_header + TRX_RSEG_HISTORY, From b0ef1b388bff28b93bfdf83a97723a56c869ca34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 21 Aug 2018 16:52:59 +0300 Subject: [PATCH 040/111] After-merge fix: Revert MDEV-15511 My conflict resolution for the script did not work out after all, and apparently I was testing a wrong version. Revert MDEV-15511 from MariaDB 10.2 for now. --- scripts/wsrep_sst_rsync.sh | 80 +++++++------------------------------- 1 file changed, 14 insertions(+), 66 deletions(-) diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 2824593a0a9..88b1352db2e 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -41,8 +41,6 @@ cleanup_joiner() kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || \ : rm -rf "$RSYNC_CONF" - rm -f "$STUNNEL_CONF" - rm -f "$STUNNEL_PID" rm -rf "$MAGIC_FILE" rm -rf "$RSYNC_PID" wsrep_log_info "Joiner cleanup done." @@ -70,7 +68,7 @@ check_pid_and_port() local port_info="$(sockstat -46lp ${rsync_port} 2>/dev/null | \ grep ":${rsync_port}")" local is_rsync="$(echo $port_info | \ - grep -wE '[[:space:]]\+(rsync|stunnel)[[:space:]]\+'"$rsync_pid" 2>/dev/null)" + grep '[[:space:]]\+rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null)" ;; *) if ! which lsof > /dev/null; then @@ -81,7 +79,7 @@ check_pid_and_port() local port_info="$(lsof -i :$rsync_port -Pn 2>/dev/null | \ grep "(LISTEN)")" local is_rsync="$(echo $port_info | \ - grep -wE '^(rsync|stunnel)[[:space:]]\+'"$rsync_pid" 2>/dev/null)" + grep -w '^rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null)" ;; esac @@ -121,12 +119,6 @@ is_local_ip() $get_addr_bin | grep "$address" > /dev/null } -STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf" -rm -f "$STUNNEL_CONF" - -STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" -rm -f "$STUNNEL_PID" - MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" rm -rf "$MAGIC_FILE" @@ -164,28 +156,9 @@ fi FILTER="-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*'" -SSTKEY=$(parse_cnf sst tkey "") -SSTCERT=$(parse_cnf sst tcert "") -STUNNEL="" -if [ -f "$SSTKEY" ] && [ -f "$SSTCERT" ] && wsrep_check_programs stunnel -then - STUNNEL="stunnel ${STUNNEL_CONF}" -fi - if [ "$WSREP_SST_OPT_ROLE" = "donor" ] then -cat << EOF > "$STUNNEL_CONF" -CApath = ${SSTCERT%/*} -foreground = yes -pid = $STUNNEL_PID -debug = warning -client = yes -connect = ${WSREP_SST_OPT_ADDR%/*} -TIMEOUTclose = 0 -verifyPeer = yes -EOF - if [ $WSREP_SST_OPT_BYPASS -eq 0 ] then @@ -247,8 +220,7 @@ EOF # first, the normal directories, so that we can detect incompatible protocol RC=0 - eval rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ + eval rsync --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT ${FILTER} "$WSREP_SST_OPT_DATA/" \ rsync://$WSREP_SST_OPT_ADDR >&2 || RC=$? @@ -271,8 +243,7 @@ EOF fi # second, we transfer InnoDB log files - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ + rsync --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '- **' "$WSREP_LOG_DIR/" \ rsync://$WSREP_SST_OPT_ADDR-log_dir >&2 || RC=$? @@ -292,8 +263,7 @@ EOF find . -maxdepth 1 -mindepth 1 -type d -not -name "lost+found" \ -print0 | xargs -I{} -0 -P $count \ - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ + rsync --owner --group --perms --links --specials \ --ignore-times --inplace --recursive --delete --quiet \ $WHOLE_FILE_OPT --exclude '*/ib_logfile*' "$WSREP_SST_OPT_DATA"/{}/ \ rsync://$WSREP_SST_OPT_ADDR/{} >&2 || RC=$? @@ -316,8 +286,7 @@ EOF echo "continue" # now server can resume updating data echo "$STATE" > "$MAGIC_FILE" - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR + rsync --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR echo "done $STATE" @@ -372,41 +341,20 @@ $SILENT path = $WSREP_LOG_DIR EOF -cat << EOF > "$STUNNEL_CONF" -key = $SSTKEY -cert = $SSTCERT -foreground = yes -pid = $STUNNEL_PID -debug = warning -client = no -[rsync] -accept = $RSYNC_PORT -exec = $(which rsync) -execargs = rsync --server --daemon --config=$RSYNC_CONF . -EOF - # rm -rf "$DATA"/ib_logfile* # we don't want old logs around readonly RSYNC_PORT=${WSREP_SST_OPT_PORT:-4444} - - if [ -z "$STUNNEL" ] + # If the IP is local listen only in it + if is_local_ip "$RSYNC_ADDR" then - # If the IP is local listen only in it - if is_local_ip "$RSYNC_ADDR" - then - rsync --daemon --no-detach --address "$RSYNC_ADDR" --port "$RSYNC_PORT" --config "$RSYNC_CONF" & - else - # Not local, possibly a NAT, listen in all interfaces - rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" & - # Overwrite address with all - RSYNC_ADDR="" - fi - RSYNC_REAL_PID=$! + rsync --daemon --no-detach --address "$RSYNC_ADDR" --port "$RSYNC_PORT" --config "$RSYNC_CONF" & else - stunnel "$STUNNEL_CONF" & - RSYNC_REAL_PID=$! - RSYNC_PID=$STUNNEL_PID + # Not local, possibly a NAT, listen in all interface + rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" & + # Overwrite address with all + RSYNC_ADDR="*" fi + RSYNC_REAL_PID=$! until check_pid_and_port "$RSYNC_PID" "$RSYNC_REAL_PID" "$RSYNC_ADDR" "$RSYNC_PORT" do From a1fd25c22bc27e58b802dd83ee48428913351180 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sat, 18 Aug 2018 22:57:20 -0700 Subject: [PATCH 041/111] MDEV-17017 Explain for query using derived table specified with a table value constructor shows wrong number of rows If the specification of a derived table contained a table value constructor then the optimizer incorrectly estimated the number of rows in the derived table. This happened because the optimizer did not take into account the number of rows in the constructor. The wrong estimate could lead to choosing inefficient execution plans. --- mysql-test/main/opt_tvc.result | 30 ++++++------ mysql-test/main/opt_tvc.test | 2 +- mysql-test/main/range.result | 6 +-- mysql-test/main/range.test | 4 ++ mysql-test/main/range_mrr_icp.result | 6 +-- mysql-test/main/table_value_constr.result | 57 +++++++++++++++++++++++ mysql-test/main/table_value_constr.test | 29 ++++++++++++ sql/sql_tvc.h | 2 + sql/sql_union.cc | 2 + 9 files changed, 115 insertions(+), 23 deletions(-) diff --git a/mysql-test/main/opt_tvc.result b/mysql-test/main/opt_tvc.result index 0ecae5bf157..fdbd932a433 100644 --- a/mysql-test/main/opt_tvc.result +++ b/mysql-test/main/opt_tvc.result @@ -486,34 +486,32 @@ a b deallocate prepare stmt; # use inside out access from tvc rows set @@in_predicate_conversion_threshold= default; -select * from t3 where a in (1,4,10); +select * from t3 where a in (1,4); a b 1 abc 1 todd 1 sm 4 yq -10 abc -explain extended select * from t3 where a in (1,4,10); +explain extended select * from t3 where a in (1,4); id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE t3 range idx idx 5 NULL 5 100.00 Using index condition +1 SIMPLE t3 range idx idx 5 NULL 4 100.00 Using index condition Warnings: -Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t3` where `test`.`t3`.`a` in (1,4,10) +Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t3` where `test`.`t3`.`a` in (1,4) set @@in_predicate_conversion_threshold= 2; -select * from t3 where a in (1,4,10); +select * from t3 where a in (1,4); a b 1 abc 1 todd 1 sm 4 yq -10 abc -explain extended select * from t3 where a in (1,4,10); +explain extended select * from t3 where a in (1,4); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY ALL distinct_key NULL NULL NULL 2 100.00 1 PRIMARY t3 ref idx idx 5 tvc_0.1 3 100.00 2 MATERIALIZED ALL NULL NULL NULL NULL 2 100.00 3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t3` semi join ((values (1),(4),(10)) `tvc_0`) where `test`.`t3`.`a` = `tvc_0`.`1` +Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t3` semi join ((values (1),(4)) `tvc_0`) where `test`.`t3`.`a` = `tvc_0`.`1` # use vectors in IN predeicate set @@in_predicate_conversion_threshold= 4; select * from t1 where (a,b) in ((1,2),(3,4)); @@ -540,9 +538,9 @@ explain extended select * from t2 where (a,b) in ((1,2),(8,9)) and (a,c) in ((1,3),(8,0),(5,1)); id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL distinct_key NULL NULL NULL 2 100.00 +1 PRIMARY ALL distinct_key NULL NULL NULL 3 100.00 1 PRIMARY t2 ALL NULL NULL NULL NULL 6 100.00 Using where; Using join buffer (flat, BNL join) -2 MATERIALIZED ALL NULL NULL NULL NULL 2 100.00 +2 MATERIALIZED ALL NULL NULL NULL NULL 3 100.00 3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: Note 1003 /* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t2` semi join ((values (1,3),(8,0),(5,1)) `tvc_0`) where `test`.`t2`.`a` = `tvc_0`.`1` and `test`.`t2`.`c` = `tvc_0`.`3` and (`tvc_0`.`1`,`test`.`t2`.`b`) in (((1,2)),((8,9))) @@ -570,7 +568,7 @@ explain extended select * from t1 where (a,b) not in ((1,2),(8,9), (5,1)); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 6 100.00 Using where -2 MATERIALIZED ALL NULL NULL NULL NULL 2 100.00 +2 MATERIALIZED ALL NULL NULL NULL NULL 3 100.00 3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: Note 1003 /* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where !<`test`.`t1`.`a`,`test`.`t1`.`b`>(((`test`.`t1`.`a`,`test`.`t1`.`b`),(`test`.`t1`.`a`,`test`.`t1`.`b`) in ( (/* select#2 */ select `tvc_0`.`1`,`tvc_0`.`2` from (values (1,2),(8,9),(5,1)) `tvc_0` ), (`test`.`t1`.`a` in on distinct_key where `test`.`t1`.`a` = ``.`1` and `test`.`t1`.`b` = ``.`2`)))) @@ -578,7 +576,7 @@ explain extended select * from t1 where (a,b) not in (select * from (values (1,2),(8,9), (5,1)) as tvc_0); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 6 100.00 Using where -2 MATERIALIZED ALL NULL NULL NULL NULL 2 100.00 +2 MATERIALIZED ALL NULL NULL NULL NULL 3 100.00 3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: Note 1003 /* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where !<`test`.`t1`.`a`,`test`.`t1`.`b`>(((`test`.`t1`.`a`,`test`.`t1`.`b`),(`test`.`t1`.`a`,`test`.`t1`.`b`) in ( (/* select#2 */ select `tvc_0`.`1`,`tvc_0`.`2` from (values (1,2),(8,9),(5,1)) `tvc_0` ), (`test`.`t1`.`a` in on distinct_key where `test`.`t1`.`a` = ``.`1` and `test`.`t1`.`b` = ``.`2`)))) @@ -592,7 +590,7 @@ explain extended select * from t1 where b < 7 and (a,b) not in ((1,2),(8,9), (5,1)); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 6 100.00 Using where -2 MATERIALIZED ALL NULL NULL NULL NULL 2 100.00 +2 MATERIALIZED ALL NULL NULL NULL NULL 3 100.00 3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: Note 1003 /* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`b` < 7 and !<`test`.`t1`.`a`,`test`.`t1`.`b`>(((`test`.`t1`.`a`,`test`.`t1`.`b`),(`test`.`t1`.`a`,`test`.`t1`.`b`) in ( (/* select#2 */ select `tvc_0`.`1`,`tvc_0`.`2` from (values (1,2),(8,9),(5,1)) `tvc_0` ), (`test`.`t1`.`a` in on distinct_key where `test`.`t1`.`a` = ``.`1` and `test`.`t1`.`b` = ``.`2`)))) @@ -608,7 +606,7 @@ explain extended select * from t2 where (a,c) not in ((1,2),(8,9), (5,1)); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 6 100.00 Using where -2 MATERIALIZED ALL NULL NULL NULL NULL 2 100.00 +2 MATERIALIZED ALL NULL NULL NULL NULL 3 100.00 3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: Note 1003 /* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t2` where !<`test`.`t2`.`a`,`test`.`t2`.`c`>(((`test`.`t2`.`a`,`test`.`t2`.`c`),(`test`.`t2`.`a`,`test`.`t2`.`c`) in ( (/* select#2 */ select `tvc_0`.`1`,`tvc_0`.`2` from (values (1,2),(8,9),(5,1)) `tvc_0` ), (`test`.`t2`.`a` in on distinct_key where `test`.`t2`.`a` = ``.`1` and `test`.`t2`.`c` = ``.`2`)))) @@ -632,7 +630,7 @@ i EXPLAIN EXTENDED SELECT * FROM t1 WHERE i IN (NULL, NULL, NULL, NULL, NULL); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 -1 PRIMARY ALL NULL NULL NULL NULL 2 100.00 Using where; FirstMatch(t1); Using join buffer (flat, BNL join) +1 PRIMARY ALL NULL NULL NULL NULL 5 100.00 Using where; FirstMatch(t1); Using join buffer (flat, BNL join) 3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: Note 1003 /* select#1 */ select `test`.`t1`.`i` AS `i` from `test`.`t1` semi join ((values (NULL),(NULL),(NULL),(NULL),(NULL)) `tvc_0`) where `test`.`t1`.`i` = `tvc_0`.`NULL` diff --git a/mysql-test/main/opt_tvc.test b/mysql-test/main/opt_tvc.test index d5c9a5cbd3d..2d06a0a0b20 100644 --- a/mysql-test/main/opt_tvc.test +++ b/mysql-test/main/opt_tvc.test @@ -255,7 +255,7 @@ deallocate prepare stmt; --echo # use inside out access from tvc rows -let $query= select * from t3 where a in (1,4,10); +let $query= select * from t3 where a in (1,4); set @@in_predicate_conversion_threshold= default; eval $query; eval explain extended $query; diff --git a/mysql-test/main/range.result b/mysql-test/main/range.result index e2996b963bc..cbf9d5bb7d9 100644 --- a/mysql-test/main/range.result +++ b/mysql-test/main/range.result @@ -1052,6 +1052,7 @@ create table t1 (a int); insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t2 (a int, key(a)); insert into t2 select 2*(A.a + 10*(B.a + 10*C.a)) from t1 A, t1 B, t1 C; +set in_predicate_conversion_threshold= 2000; set @a="select * from t2 force index (a) where a NOT IN(0"; select count(*) from (select @a:=concat(@a, ',', a) from t2 ) Z; count(*) @@ -1062,15 +1063,14 @@ set @b= concat("explain ", @a); prepare stmt1 from @b; execute stmt1; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t2 index NULL a 5 NULL 1003 Using where; Using index -2 MATERIALIZED ALL NULL NULL NULL NULL 2 -3 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +1 SIMPLE t2 index a a 5 NULL 1003 Using where; Using index prepare stmt1 from @a; execute stmt1; a 11 13 15 +set in_predicate_conversion_threshold= default; drop table t1, t2; CREATE TABLE t1 ( id int NOT NULL DEFAULT '0', diff --git a/mysql-test/main/range.test b/mysql-test/main/range.test index 36e0e32b28b..43b5b18435e 100644 --- a/mysql-test/main/range.test +++ b/mysql-test/main/range.test @@ -870,6 +870,8 @@ insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t2 (a int, key(a)); insert into t2 select 2*(A.a + 10*(B.a + 10*C.a)) from t1 A, t1 B, t1 C; +set in_predicate_conversion_threshold= 2000; + set @a="select * from t2 force index (a) where a NOT IN(0"; select count(*) from (select @a:=concat(@a, ',', a) from t2 ) Z; set @a=concat(@a, ')'); @@ -884,6 +886,8 @@ execute stmt1; prepare stmt1 from @a; execute stmt1; +set in_predicate_conversion_threshold= default; + drop table t1, t2; # diff --git a/mysql-test/main/range_mrr_icp.result b/mysql-test/main/range_mrr_icp.result index 629d183bee7..483957ff39e 100644 --- a/mysql-test/main/range_mrr_icp.result +++ b/mysql-test/main/range_mrr_icp.result @@ -1054,6 +1054,7 @@ create table t1 (a int); insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t2 (a int, key(a)); insert into t2 select 2*(A.a + 10*(B.a + 10*C.a)) from t1 A, t1 B, t1 C; +set in_predicate_conversion_threshold= 2000; set @a="select * from t2 force index (a) where a NOT IN(0"; select count(*) from (select @a:=concat(@a, ',', a) from t2 ) Z; count(*) @@ -1064,15 +1065,14 @@ set @b= concat("explain ", @a); prepare stmt1 from @b; execute stmt1; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t2 index NULL a 5 NULL 1003 Using where; Using index -2 MATERIALIZED ALL NULL NULL NULL NULL 2 -3 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +1 SIMPLE t2 index a a 5 NULL 1003 Using where; Using index prepare stmt1 from @a; execute stmt1; a 11 13 15 +set in_predicate_conversion_threshold= default; drop table t1, t2; CREATE TABLE t1 ( id int NOT NULL DEFAULT '0', diff --git a/mysql-test/main/table_value_constr.result b/mysql-test/main/table_value_constr.result index 9e0a0968932..b0b0fa81664 100644 --- a/mysql-test/main/table_value_constr.result +++ b/mysql-test/main/table_value_constr.result @@ -2097,3 +2097,60 @@ v # with t as (values (),()) select 1 from t; ERROR HY000: Row with no elements is not allowed in table value constructor in this context +# +# MDEV-17017: TVC in derived table +# +create table t1 (a int); +insert into t1 values (9), (3), (2); +select * from (values (7), (5), (8), (1), (3), (8), (1)) t; +7 +7 +5 +8 +1 +3 +8 +1 +explain select * from (values (7), (5), (8), (1), (3), (8), (1)) t; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 7 +2 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +select * from (values (1,11), (7,77), (3,31), (4,42)) t; +1 11 +1 11 +7 77 +3 31 +4 42 +explain select * from (values (1,11), (7,77), (3,31), (4,42)) t; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 4 +2 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +select * from (values (7), (5), (8), (1) union values (3), (8), (1)) t; +7 +7 +5 +8 +1 +3 +explain select * from (values (7), (5), (8), (1) union values (3), (8), (1)) t; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 7 +2 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +3 UNION NULL NULL NULL NULL NULL NULL NULL No tables used +NULL UNION RESULT ALL NULL NULL NULL NULL NULL +select * from (values (7), (5), (8), (1) union select * from t1) t; +7 +7 +5 +8 +1 +9 +3 +2 +explain select * from (values (7), (5), (8), (1) union select * from t1) t; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 7 +2 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +3 UNION t1 ALL NULL NULL NULL NULL 3 +NULL UNION RESULT ALL NULL NULL NULL NULL NULL +drop table t1; diff --git a/mysql-test/main/table_value_constr.test b/mysql-test/main/table_value_constr.test index eb5ea59f829..eb30f00fa91 100644 --- a/mysql-test/main/table_value_constr.test +++ b/mysql-test/main/table_value_constr.test @@ -1075,3 +1075,32 @@ DELIMITER ;| --error ER_EMPTY_ROW_IN_TVC with t as (values (),()) select 1 from t; + +--echo # +--echo # MDEV-17017: TVC in derived table +--echo # + +create table t1 (a int); +insert into t1 values (9), (3), (2); + +let $q1= +select * from (values (7), (5), (8), (1), (3), (8), (1)) t; +eval $q1; +eval explain $q1; + +let $q2= +select * from (values (1,11), (7,77), (3,31), (4,42)) t; +eval $q2; +eval explain $q2; + +let $q3= +select * from (values (7), (5), (8), (1) union values (3), (8), (1)) t; +eval $q3; +eval explain $q3; + +let $q4= +select * from (values (7), (5), (8), (1) union select * from t1) t; +eval $q4; +eval explain $q4; + +drop table t1; diff --git a/sql/sql_tvc.h b/sql/sql_tvc.h index 420311cccb2..128cc883dd8 100644 --- a/sql/sql_tvc.h +++ b/sql/sql_tvc.h @@ -50,6 +50,8 @@ public: have_query_plan(QEP_NOT_PRESENT_YET), explain(0), select_options(select_options_arg) { }; + + ha_rows get_records() { return lists_of_values.elements; } bool prepare(THD *thd_arg, SELECT_LEX *sl, select_result *tmp_result, diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 12035326872..c8bf9bda57b 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -1321,6 +1321,8 @@ bool st_select_lex_unit::optimize() thd->lex->current_select= lex_select_save; DBUG_RETURN(TRUE); } + if (derived) + sl->increase_derived_records(sl->tvc->get_records()); continue; } thd->lex->current_select= sl; From c43d11b96e27f25d248f4740a2274f1bfb1d5845 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Tue, 21 Aug 2018 12:01:25 -0700 Subject: [PATCH 042/111] MDEV-16930 Crash when VALUES in derived table contains expressions This patch always provides columns of the temporary table used for materialization of a table value constructor with some names. Before this patch these names were always borrowed from the items of the first row of the table value constructor. When this row contained expressions and expressions were not named then it could cause different kinds of problems. In particular if the TVC is used as the specification of a derived table this could cause a crash. The names given to the expressions used in a TVC are the same as those given to the columns of the result set from the corresponding SELECT. --- mysql-test/main/table_value_constr.result | 35 +++++++++++++++++++ mysql-test/main/table_value_constr.test | 21 ++++++++++++ sql/sql_yacc.yy | 42 ++++++++++++++++++++++- sql/sql_yacc_ora.yy | 42 ++++++++++++++++++++++- 4 files changed, 138 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/table_value_constr.result b/mysql-test/main/table_value_constr.result index b0b0fa81664..1d485af4a4d 100644 --- a/mysql-test/main/table_value_constr.result +++ b/mysql-test/main/table_value_constr.result @@ -2154,3 +2154,38 @@ id select_type table type possible_keys key key_len ref rows Extra 3 UNION t1 ALL NULL NULL NULL NULL 3 NULL UNION RESULT ALL NULL NULL NULL NULL NULL drop table t1; +# +# MDEV-16930: expression in the first row of TVC specifying derived table +# +SELECT 1 + 1, 2, "abc"; +1 + 1 2 abc +2 2 abc +SELECT * FROM (SELECT 1 + 1, 2, "abc") t; +1 + 1 2 abc +2 2 abc +WITH cte AS (SELECT 1 + 1, 2, "abc") SELECT * FROM cte; +1 + 1 2 abc +2 2 abc +SELECT 1 + 1, 2, "abc" UNION SELECT 3+4, 3, "abc"; +1 + 1 2 abc +2 2 abc +7 3 abc +CREATE VIEW v1 AS SELECT 1 + 1, 2, "abc"; +SELECT * FROM v1; +1 + 1 2 abc +2 2 abc +DROP VIEW v1; +VALUES(1 + 1,2,"abc"); +1 + 1 2 abc +2 2 abc +SELECT * FROM (VALUES(1 + 1,2,"abc")) t; +1 + 1 2 abc +2 2 abc +PREPARE stmt FROM "SELECT * FROM (VALUES(1 + 1,2,'abc')) t"; +EXECUTE stmt; +1 + 1 2 abc +2 2 abc +EXECUTE stmt; +1 + 1 2 abc +2 2 abc +DEALLOCATE PREPARE stmt; diff --git a/mysql-test/main/table_value_constr.test b/mysql-test/main/table_value_constr.test index eb30f00fa91..5df40d1047c 100644 --- a/mysql-test/main/table_value_constr.test +++ b/mysql-test/main/table_value_constr.test @@ -1104,3 +1104,24 @@ eval $q4; eval explain $q4; drop table t1; + +--echo # +--echo # MDEV-16930: expression in the first row of TVC specifying derived table +--echo # + +SELECT 1 + 1, 2, "abc"; +SELECT * FROM (SELECT 1 + 1, 2, "abc") t; +WITH cte AS (SELECT 1 + 1, 2, "abc") SELECT * FROM cte; +SELECT 1 + 1, 2, "abc" UNION SELECT 3+4, 3, "abc"; +CREATE VIEW v1 AS SELECT 1 + 1, 2, "abc"; +SELECT * FROM v1; +DROP VIEW v1; + +VALUES(1 + 1,2,"abc"); +SELECT * FROM (VALUES(1 + 1,2,"abc")) t; +PREPARE stmt FROM "SELECT * FROM (VALUES(1 + 1,2,'abc')) t"; +EXECUTE stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + + diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index f915895a260..1ec731781a5 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -2044,6 +2044,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); preload_list preload_list_or_parts preload_keys preload_keys_parts select_item_list select_item values_list no_braces opt_limit_clause delete_limit_clause fields opt_values values + no_braces_with_names opt_values_with_names values_with_names procedure_list procedure_list2 procedure_item field_def handler opt_generated_always opt_ignore opt_column opt_restrict @@ -13247,7 +13248,7 @@ insert_values: values_list: values_list ',' no_braces - | no_braces + | no_braces_with_names ; ident_eq_list: @@ -13300,11 +13301,31 @@ no_braces: } ; +no_braces_with_names: + '(' + { + if (unlikely(!(Lex->insert_list= new (thd->mem_root) List_item))) + MYSQL_YYABORT; + } + opt_values_with_names ')' + { + LEX *lex=Lex; + if (unlikely(lex->many_values.push_back(lex->insert_list, + thd->mem_root))) + MYSQL_YYABORT; + } + ; + opt_values: /* empty */ {} | values ; +opt_values_with_names: + /* empty */ {} + | values_with_names + ; + values: values ',' expr_or_default { @@ -13318,6 +13339,25 @@ values: } ; +values_with_names: + values_with_names ',' remember_name expr_or_default remember_end + { + if (unlikely(Lex->insert_list->push_back($4, thd->mem_root))) + MYSQL_YYABORT; + // give some name in case of using in table value constuctor (TVC) + if (!$4->name.str) + $4->set_name(thd, $3, (uint) ($5 - $3), thd->charset()); + } + | remember_name expr_or_default remember_end + { + if (unlikely(Lex->insert_list->push_back($2, thd->mem_root))) + MYSQL_YYABORT; + // give some name in case of using in table value constuctor (TVC) + if (!$2->name.str) + $2->set_name(thd, $1, (uint) ($3 - $1), thd->charset()); + } + ; + expr_or_default: expr { $$= $1;} | DEFAULT diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index d95e87c524a..a6446180fc0 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -1450,6 +1450,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); assign_to_keycache_parts preload_list preload_list_or_parts preload_keys preload_keys_parts select_item_list select_item values_list no_braces + no_braces_with_names opt_values_with_names values_with_names opt_limit_clause delete_limit_clause fields opt_values values procedure_list procedure_list2 procedure_item field_def handler opt_generated_always @@ -13402,7 +13403,7 @@ insert_values: values_list: values_list ',' no_braces - | no_braces + | no_braces_with_names ; ident_eq_list: @@ -13455,11 +13456,31 @@ no_braces: } ; +no_braces_with_names: + '(' + { + if (unlikely(!(Lex->insert_list= new (thd->mem_root) List_item))) + MYSQL_YYABORT; + } + opt_values_with_names ')' + { + LEX *lex=Lex; + if (unlikely(lex->many_values.push_back(lex->insert_list, + thd->mem_root))) + MYSQL_YYABORT; + } + ; + opt_values: /* empty */ {} | values ; +opt_values_with_names: + /* empty */ {} + | values_with_names + ; + values: values ',' expr_or_default { @@ -13473,6 +13494,25 @@ values: } ; +values_with_names: + values_with_names ',' remember_name expr_or_default remember_end + { + if (unlikely(Lex->insert_list->push_back($4, thd->mem_root))) + MYSQL_YYABORT; + // give some name in case of using in table value constuctor (TVC) + if (!$4->name.str) + $4->set_name(thd, $3, (uint) ($5 - $3), thd->charset()); + } + | remember_name expr_or_default remember_end + { + if (unlikely(Lex->insert_list->push_back($2, thd->mem_root))) + MYSQL_YYABORT; + // give some name in case of using in table value constuctor (TVC) + if (!$2->name.str) + $2->set_name(thd, $1, (uint) ($3 - $1), thd->charset()); + } + ; + expr_or_default: expr { $$= $1;} | DEFAULT From 28461fe5ef419a06b93cae9358918e6cf40a5b07 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 22 Aug 2018 22:06:38 +0300 Subject: [PATCH 043/111] MDEV-16330 Allow instant change of WITH SYSTEM VERSIONING column attribute cleanup change_fields_versioning_try(): use innodb_col_no() instead of a manual loop change_fields_versioning_cache(): use innodb_col_no() instead of a manual loop --- storage/innobase/handler/handler0alter.cc | 48 +++-------------------- 1 file changed, 5 insertions(+), 43 deletions(-) diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 3f31f253b3f..2f4d49d79a3 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -8491,36 +8491,17 @@ change_fields_versioning_try( DBUG_RETURN(false); } - uint virtual_count = 0; - List_iterator_fast it( ha_alter_info->alter_info->create_list); - for (uint i = 0; i < table->s->fields; i++) { - const Field* field = table->field[i]; - - if (innobase_is_v_fld(field)) { - virtual_count++; - continue; - } - - const Create_field* create_field = NULL; - while (const Create_field* cf = it++) { - if (cf->field == field) { - create_field = cf; - break; - } - } - it.rewind(); - DBUG_ASSERT(create_field); - + while (const Create_field* create_field = it++) { if (create_field->versioning == Column_definition::VERSIONING_NOT_SET) { continue; } const dict_table_t* new_table = ctx->new_table; - ulint pos = i - virtual_count; + const uint pos = innodb_col_no(create_field->field); const dict_col_t* col = dict_table_get_nth_col(new_table, pos); DBUG_ASSERT(!col->vers_sys_start()); @@ -8560,31 +8541,12 @@ change_fields_versioning_cache( DBUG_ASSERT(ctx); DBUG_ASSERT(ha_alter_info->handler_flags & ALTER_COLUMN_UNVERSIONED); - uint virtual_count = 0; - List_iterator_fast it( ha_alter_info->alter_info->create_list); - for (uint i = 0; i < table->s->fields; i++) { - const Field* field = table->field[i]; - - if (innobase_is_v_fld(field)) { - virtual_count++; - continue; - } - - const Create_field* create_field = NULL; - while (const Create_field* cf = it++) { - if (cf->field == field) { - create_field = cf; - break; - } - } - it.rewind(); - DBUG_ASSERT(create_field); - - dict_col_t* col - = dict_table_get_nth_col(ctx->new_table, i - virtual_count); + while (const Create_field* create_field = it++) { + dict_col_t* col = dict_table_get_nth_col( + ctx->new_table, innodb_col_no(create_field->field)); if (create_field->versioning == Column_definition::WITHOUT_VERSIONING) { From 5d650d366dbcbd49679bf8b0884b63056053fb44 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 22 Aug 2018 22:06:40 +0200 Subject: [PATCH 044/111] MDEV-16961 Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' failed upon concurrent DELETE and DDL with virtual blob column After iterating all fields and setting PART_INDIRECT_KEY_FLAG as necessary, TABLE::mark_columns_used_by_virtual_fields() remembers in TABLE_SHARE that this operation was done and need not be repeated. But as the flag is set in TABLE_SHARE, PART_INDIRECT_KEY_FLAG must be set in TABLE_SHARE::field[], not only in TABLE::field[]. Otherwise all new TABLEs opened from this TABLE_SHARE will never have it. --- mysql-test/suite/vcol/r/index.result | 10 ++++++++++ mysql-test/suite/vcol/t/index.test | 16 ++++++++++++++++ sql/table.cc | 3 +++ 3 files changed, 29 insertions(+) diff --git a/mysql-test/suite/vcol/r/index.result b/mysql-test/suite/vcol/r/index.result index 8860a728bd1..6ab9aa5eb29 100644 --- a/mysql-test/suite/vcol/r/index.result +++ b/mysql-test/suite/vcol/r/index.result @@ -89,3 +89,13 @@ pk left(c, 10) length(c) i 1 bar bar ba 60000 11 drop table t1; disconnect c1; +CREATE TABLE t1 (b BLOB, vb TEXT AS (b) PERSISTENT, KEY(vb(64))) ENGINE=InnoDB; +INSERT INTO t1 (b) VALUES ('foo'); +connect con1,localhost,root,,test; +CREATE TABLE t2 LIKE t1; +connection default; +DELETE FROM t1; +connection con1; +disconnect con1; +connection default; +DROP TABLE t1, t2; diff --git a/mysql-test/suite/vcol/t/index.test b/mysql-test/suite/vcol/t/index.test index 72eed0a8a40..b93c337fab6 100644 --- a/mysql-test/suite/vcol/t/index.test +++ b/mysql-test/suite/vcol/t/index.test @@ -79,3 +79,19 @@ commit; select pk, left(c, 10), length(c), i from t1; drop table t1; disconnect c1; + +# +# MDEV-16961 Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' failed upon concurrent DELETE and DDL with virtual blob column +# + +CREATE TABLE t1 (b BLOB, vb TEXT AS (b) PERSISTENT, KEY(vb(64))) ENGINE=InnoDB; +INSERT INTO t1 (b) VALUES ('foo'); +--connect (con1,localhost,root,,test) +--send CREATE TABLE t2 LIKE t1 +--connection default +DELETE FROM t1; +--connection con1 +--reap +--disconnect con1 +--connection default +DROP TABLE t1, t2; diff --git a/sql/table.cc b/sql/table.cc index 201901fd46f..efcca431e93 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -6736,7 +6736,10 @@ void TABLE::mark_columns_used_by_virtual_fields(void) for (uint i= 0 ; i < s->fields ; i++) { if (bitmap_is_set(&tmp_set, i)) + { + s->field[i]->flags|= PART_INDIRECT_KEY_FLAG; field[i]->flags|= PART_INDIRECT_KEY_FLAG; + } } bitmap_clear_all(&tmp_set); } From f7154242b8807fdaa74ed0b89c3de8c35ee35edb Mon Sep 17 00:00:00 2001 From: Ming Lin Date: Thu, 16 Aug 2018 10:28:29 -0700 Subject: [PATCH 045/111] MDEV-16703: Update AUTO_INCREMENT in the UPDATE statement Currently RocksDB engine doesn't update AUTO_INCREMENT in the UPDATE statement. For example, CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; INSERT INTO t1 (a) VALUES (1); UPDATE t1 SET pk = 3; ==> AUTO_INCREMENT should be updated to 4. Without this fix, it hits the Assertion `dd_val >= last_val' failed in myrocks::ha_rocksdb::load_auto_incr_value_from_index. --- storage/rocksdb/ha_rocksdb.cc | 2 +- .../rocksdb/mysql-test/rocksdb/r/autoinc_vars.result | 8 ++++++++ storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test | 10 ++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 48108f9d409..2b6ba2616d8 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -9553,7 +9553,7 @@ int ha_rocksdb::update_pk(const Rdb_key_def &kd, } } - if (table->next_number_field) { + if (table->found_next_number_field) { update_auto_incr_val_from_field(); } diff --git a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result index 8cc9b070a70..0c496227006 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result @@ -150,3 +150,11 @@ CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; INSERT INTO t0 VALUES(0); ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +#--------------------------------------------------------------- +# MDEV-16703 Assertion failed in load_auto_incr_value_from_index +#--------------------------------------------------------------- +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test index 9d7f0365d07..86a0a7f2a53 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test @@ -116,3 +116,13 @@ CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; INSERT INTO t0 VALUES(0); ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; + +--echo #--------------------------------------------------------------- +--echo # MDEV-16703 Assertion failed in load_auto_incr_value_from_index +--echo #--------------------------------------------------------------- + +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; From 9a815401c6c1972745432ff93cbc8f55cae30d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 23 Aug 2018 13:11:11 +0300 Subject: [PATCH 046/111] MDEV-17043 Purge of indexed virtual columns may cause hang on table-rebuilding DDL When a table is renamed to an internal #sql2 or #sql-ib name during a table-rebuilding DDL operation such as OPTIMIZE TABLE or ALTER TABLE, and shortly after that a purge operation in an index on virtual columns is attempted, the operation could fail, but purge would fail to release the table reference. innodb_acquire_mdl(): Release the reference if the table name is not valid for acquiring a meta-data lock (MDL). innodb_find_table_for_vc(): Add a debug assertion if the table name is not valid. This code path is for DML execution. The table should have a valid name for executing DML, and furthermore a MDL will prevent the table from being renamed. row_vers_build_clust_v_col(): Add a debug assertion that both indexes must belong to the same table. --- storage/innobase/handler/ha_innodb.cc | 2 ++ storage/innobase/row/row0vers.cc | 1 + 2 files changed, 3 insertions(+) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index c3a6bd663b3..35e091e0f11 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -21425,6 +21425,7 @@ static TABLE* innodb_acquire_mdl(THD* thd, dict_table_t* table) if (!table_name_parse(table->name, db_buf, tbl_buf, db_buf_len, tbl_buf_len)) { + table->release(); return NULL; } @@ -21507,6 +21508,7 @@ static TABLE* innodb_find_table_for_vc(THD* thd, dict_table_t* table) if (!table_name_parse(table->name, db_buf, tbl_buf, db_buf_len, tbl_buf_len)) { + ut_ad(!"invalid table name"); return NULL; } diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 24acd04c84b..811654fdef8 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -456,6 +456,7 @@ row_vers_build_clust_v_col( byte* record= 0; ut_ad(dict_index_has_virtual(index)); + ut_ad(index->table == clust_index->table); if (vcol_info != NULL) { vcol_info->set_used(); From c164d0cc6226db586ca62a10b925439ca8fdf086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 23 Aug 2018 17:02:50 +0300 Subject: [PATCH 047/111] fil_name_process(): Remove unused return value --- storage/innobase/log/log0recv.cc | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index c623b884628..86c415599bb 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -194,11 +194,9 @@ void (*log_file_op)(ulint space_id, const byte* flags, @param[in,out] name file name @param[in] len length of the file name @param[in] space_id the tablespace ID -@param[in] deleted whether this is a MLOG_FILE_DELETE record -@retval true if able to process file successfully. -@retval false if unable to process the file */ +@param[in] deleted whether this is a MLOG_FILE_DELETE record */ static -bool +void fil_name_process( char* name, ulint len, @@ -206,15 +204,13 @@ fil_name_process( bool deleted) { if (srv_operation == SRV_OPERATION_BACKUP) { - return true; + return; } ut_ad(srv_operation == SRV_OPERATION_NORMAL || srv_operation == SRV_OPERATION_RESTORE || srv_operation == SRV_OPERATION_RESTORE_EXPORT); - bool processed = true; - /* We will also insert space=NULL into the map, so that further checks can ensure that a MLOG_FILE_NAME record was scanned before applying any page records for the space_id. */ @@ -261,7 +257,6 @@ fil_name_process( << f.name << "' and '" << name << "'." " You must delete one of them."; recv_sys->found_corrupt_fs = true; - processed = false; } break; @@ -314,7 +309,6 @@ fil_name_process( " remove the .ibd file, you can set" " --innodb_force_recovery."; recv_sys->found_corrupt_fs = true; - processed = false; break; } @@ -325,7 +319,6 @@ fil_name_process( break; } } - return(processed); } /** Parse or process a MLOG_FILE_* record. From 6b9dd66f0770077b2573de5d16671e7463f8c5d5 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Thu, 23 Aug 2018 19:30:26 +0300 Subject: [PATCH 048/111] Move the testcase for BUG#21282 to a file that includes have_debug.inc The testcase needs to set in_predicate_conversion_threshold which is only available in debug builds (this is subject to further discussion). --- mysql-test/main/range.result | 24 ------------------------ mysql-test/main/range.test | 28 ---------------------------- mysql-test/main/range_debug.result | 24 ++++++++++++++++++++++++ mysql-test/main/range_debug.test | 30 ++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 52 deletions(-) create mode 100644 mysql-test/main/range_debug.result create mode 100644 mysql-test/main/range_debug.test diff --git a/mysql-test/main/range.result b/mysql-test/main/range.result index cbf9d5bb7d9..052ea09d5ac 100644 --- a/mysql-test/main/range.result +++ b/mysql-test/main/range.result @@ -1048,30 +1048,6 @@ select a, hex(filler) from t1 where a not between 'b' and 'b'; a hex(filler) a 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 drop table t1,t2,t3; -create table t1 (a int); -insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); -create table t2 (a int, key(a)); -insert into t2 select 2*(A.a + 10*(B.a + 10*C.a)) from t1 A, t1 B, t1 C; -set in_predicate_conversion_threshold= 2000; -set @a="select * from t2 force index (a) where a NOT IN(0"; -select count(*) from (select @a:=concat(@a, ',', a) from t2 ) Z; -count(*) -1000 -set @a=concat(@a, ')'); -insert into t2 values (11),(13),(15); -set @b= concat("explain ", @a); -prepare stmt1 from @b; -execute stmt1; -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 index a a 5 NULL 1003 Using where; Using index -prepare stmt1 from @a; -execute stmt1; -a -11 -13 -15 -set in_predicate_conversion_threshold= default; -drop table t1, t2; CREATE TABLE t1 ( id int NOT NULL DEFAULT '0', b int NOT NULL DEFAULT '0', diff --git a/mysql-test/main/range.test b/mysql-test/main/range.test index 43b5b18435e..7006f15e320 100644 --- a/mysql-test/main/range.test +++ b/mysql-test/main/range.test @@ -862,34 +862,6 @@ select a, hex(filler) from t1 where a not between 'b' and 'b'; drop table t1,t2,t3; -# -# BUG#21282 -# -create table t1 (a int); -insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); -create table t2 (a int, key(a)); -insert into t2 select 2*(A.a + 10*(B.a + 10*C.a)) from t1 A, t1 B, t1 C; - -set in_predicate_conversion_threshold= 2000; - -set @a="select * from t2 force index (a) where a NOT IN(0"; -select count(*) from (select @a:=concat(@a, ',', a) from t2 ) Z; -set @a=concat(@a, ')'); - -insert into t2 values (11),(13),(15); - -set @b= concat("explain ", @a); - -prepare stmt1 from @b; -execute stmt1; - -prepare stmt1 from @a; -execute stmt1; - -set in_predicate_conversion_threshold= default; - -drop table t1, t2; - # # Bug #18165: range access for BETWEEN with a constant for the first argument # diff --git a/mysql-test/main/range_debug.result b/mysql-test/main/range_debug.result new file mode 100644 index 00000000000..5597671071e --- /dev/null +++ b/mysql-test/main/range_debug.result @@ -0,0 +1,24 @@ +create table t1 (a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t2 (a int, key(a)); +insert into t2 select 2*(A.a + 10*(B.a + 10*C.a)) from t1 A, t1 B, t1 C; +set in_predicate_conversion_threshold= 2000; +set @a="select * from t2 force index (a) where a NOT IN(0"; +select count(*) from (select @a:=concat(@a, ',', a) from t2 ) Z; +count(*) +1000 +set @a=concat(@a, ')'); +insert into t2 values (11),(13),(15); +set @b= concat("explain ", @a); +prepare stmt1 from @b; +execute stmt1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 index a a 5 NULL 1003 Using where; Using index +prepare stmt1 from @a; +execute stmt1; +a +11 +13 +15 +set in_predicate_conversion_threshold= default; +drop table t1, t2; diff --git a/mysql-test/main/range_debug.test b/mysql-test/main/range_debug.test new file mode 100644 index 00000000000..ef331cdc84f --- /dev/null +++ b/mysql-test/main/range_debug.test @@ -0,0 +1,30 @@ +source include/have_debug.inc; +# +# BUG#21282 +# +create table t1 (a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t2 (a int, key(a)); +insert into t2 select 2*(A.a + 10*(B.a + 10*C.a)) from t1 A, t1 B, t1 C; + +set in_predicate_conversion_threshold= 2000; + +set @a="select * from t2 force index (a) where a NOT IN(0"; +select count(*) from (select @a:=concat(@a, ',', a) from t2 ) Z; +set @a=concat(@a, ')'); + +insert into t2 values (11),(13),(15); + +set @b= concat("explain ", @a); + +prepare stmt1 from @b; +execute stmt1; + +prepare stmt1 from @a; +execute stmt1; + +set in_predicate_conversion_threshold= default; + +drop table t1, t2; + + From b4cf8557e3821ff3220a5ec62ff937a1f96793e9 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 23 Aug 2018 14:39:38 -0700 Subject: [PATCH 049/111] Corrected test results after the last change in range.test --- mysql-test/main/range_mrr_icp.result | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/mysql-test/main/range_mrr_icp.result b/mysql-test/main/range_mrr_icp.result index 483957ff39e..9ce6f8bf5f0 100644 --- a/mysql-test/main/range_mrr_icp.result +++ b/mysql-test/main/range_mrr_icp.result @@ -1050,30 +1050,6 @@ select a, hex(filler) from t1 where a not between 'b' and 'b'; a hex(filler) a 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 drop table t1,t2,t3; -create table t1 (a int); -insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); -create table t2 (a int, key(a)); -insert into t2 select 2*(A.a + 10*(B.a + 10*C.a)) from t1 A, t1 B, t1 C; -set in_predicate_conversion_threshold= 2000; -set @a="select * from t2 force index (a) where a NOT IN(0"; -select count(*) from (select @a:=concat(@a, ',', a) from t2 ) Z; -count(*) -1000 -set @a=concat(@a, ')'); -insert into t2 values (11),(13),(15); -set @b= concat("explain ", @a); -prepare stmt1 from @b; -execute stmt1; -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 index a a 5 NULL 1003 Using where; Using index -prepare stmt1 from @a; -execute stmt1; -a -11 -13 -15 -set in_predicate_conversion_threshold= default; -drop table t1, t2; CREATE TABLE t1 ( id int NOT NULL DEFAULT '0', b int NOT NULL DEFAULT '0', From 2c76653849c1729bedc213b233089bf361c82e12 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 23 Aug 2018 17:43:54 -0700 Subject: [PATCH 050/111] Added test cases for MDEV-17017 and MDEV-16930 into compat/oracle --- .../compat/oracle/r/table_value_constr.result | 84 +++++++++++++++++++ .../compat/oracle/t/table_value_constr.test | 44 ++++++++++ 2 files changed, 128 insertions(+) diff --git a/mysql-test/suite/compat/oracle/r/table_value_constr.result b/mysql-test/suite/compat/oracle/r/table_value_constr.result index 31dcecfdb18..18fce086f6e 100644 --- a/mysql-test/suite/compat/oracle/r/table_value_constr.result +++ b/mysql-test/suite/compat/oracle/r/table_value_constr.result @@ -2099,3 +2099,87 @@ v # with t as (values (),()) select 1 from t; ERROR HY000: Row with no elements is not allowed in table value constructor in this context +# +# MDEV-17017: TVC in derived table +# +create table t1 (a int); +insert into t1 values (9), (3), (2); +select * from (values (7), (5), (8), (1), (3), (8), (1)) t; +7 +7 +5 +8 +1 +3 +8 +1 +explain select * from (values (7), (5), (8), (1), (3), (8), (1)) t; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 7 +2 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +select * from (values (1,11), (7,77), (3,31), (4,42)) t; +1 11 +1 11 +7 77 +3 31 +4 42 +explain select * from (values (1,11), (7,77), (3,31), (4,42)) t; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 4 +2 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +select * from (values (7), (5), (8), (1) union values (3), (8), (1)) t; +7 +7 +5 +8 +1 +3 +explain select * from (values (7), (5), (8), (1) union values (3), (8), (1)) t; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 7 +2 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +3 UNION NULL NULL NULL NULL NULL NULL NULL No tables used +NULL UNION RESULT ALL NULL NULL NULL NULL NULL +select * from (values (7), (5), (8), (1) union select * from t1) t; +7 +7 +5 +8 +1 +9 +3 +2 +explain select * from (values (7), (5), (8), (1) union select * from t1) t; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 7 +2 DERIVED NULL NULL NULL NULL NULL NULL NULL No tables used +3 UNION t1 ALL NULL NULL NULL NULL 3 +NULL UNION RESULT ALL NULL NULL NULL NULL NULL +drop table t1; +# +# MDEV-16930: expression in the first row of TVC specifying derived table +# +SELECT 1 + 1, 2, 'abc'; +1 + 1 2 abc +2 2 abc +SELECT * FROM (SELECT 1 + 1, 2, 'abc') t; +1 + 1 2 abc +2 2 abc +WITH cte AS (SELECT 1 + 1, 2, 'abc') SELECT * FROM cte; +1 + 1 2 abc +2 2 abc +SELECT 1 + 1, 2, 'abc' UNION SELECT 3+4, 3, 'abc'; +1 + 1 2 abc +2 2 abc +7 3 abc +CREATE VIEW v1 AS SELECT 1 + 1, 2, 'abc'; +SELECT * FROM v1; +1 + 1 2 abc +2 2 abc +DROP VIEW v1; +VALUES(1 + 1,2,'abc'); +1 + 1 2 abc +2 2 abc +SELECT * FROM (VALUES(1 + 1,2,'abc')) t; +1 + 1 2 abc +2 2 abc diff --git a/mysql-test/suite/compat/oracle/t/table_value_constr.test b/mysql-test/suite/compat/oracle/t/table_value_constr.test index 66519e93a36..37d25218664 100644 --- a/mysql-test/suite/compat/oracle/t/table_value_constr.test +++ b/mysql-test/suite/compat/oracle/t/table_value_constr.test @@ -1081,3 +1081,47 @@ DELIMITER ;| --error ER_EMPTY_ROW_IN_TVC with t as (values (),()) select 1 from t; + +--echo # +--echo # MDEV-17017: TVC in derived table +--echo # + +create table t1 (a int); +insert into t1 values (9), (3), (2); + +let $q1= +select * from (values (7), (5), (8), (1), (3), (8), (1)) t; +eval $q1; +eval explain $q1; + +let $q2= +select * from (values (1,11), (7,77), (3,31), (4,42)) t; +eval $q2; +eval explain $q2; + +let $q3= +select * from (values (7), (5), (8), (1) union values (3), (8), (1)) t; +eval $q3; +eval explain $q3; + +let $q4= +select * from (values (7), (5), (8), (1) union select * from t1) t; +eval $q4; +eval explain $q4; + +drop table t1; + +--echo # +--echo # MDEV-16930: expression in the first row of TVC specifying derived table +--echo # + +SELECT 1 + 1, 2, 'abc'; +SELECT * FROM (SELECT 1 + 1, 2, 'abc') t; +WITH cte AS (SELECT 1 + 1, 2, 'abc') SELECT * FROM cte; +SELECT 1 + 1, 2, 'abc' UNION SELECT 3+4, 3, 'abc'; +CREATE VIEW v1 AS SELECT 1 + 1, 2, 'abc'; +SELECT * FROM v1; +DROP VIEW v1; + +VALUES(1 + 1,2,'abc'); +SELECT * FROM (VALUES(1 + 1,2,'abc')) t; From 1b4c5b7327e84f9c9dd86d72b5ae706d9e82d9b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 24 Aug 2018 09:38:52 +0300 Subject: [PATCH 051/111] MDEV-16868 Same query gives different results An INSERT into a temporary table would fail to set the index page as modified. If there were no other write operations (such as UPDATE or DELETE) to the page, and the page was evicted, we would read back the old contents of the page, causing corruption or loss of data. page_cur_insert_rec_write_log(): Call mtr_t::set_modified() for temporary tables. Normally this is part of the mlog_open() call, but the mlog_open() call was only present in debug builds. This regression was caused by commit 48192f963a3a85a5127da5cc5cf485f07d72bc9d which was preparation for MDEV-11369 and supposed to affect debug builds only. Thanks to Thirunarayanan Balathandayuthapani for debugging. --- storage/innobase/page/page0cur.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 3aea43bfba0..6bdb7f56842 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -857,7 +857,8 @@ page_cur_insert_rec_write_log( ulint i; if (dict_table_is_temporary(index->table)) { - ut_ad(!mlog_open(mtr, 0)); + mtr->set_modified(); + ut_ad(mtr->get_log_mode() == MTR_LOG_NO_REDO); return; } From 6c6ca907ee84e3f129bc33be8202c0fee2f29196 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 24 Aug 2018 00:51:47 -0700 Subject: [PATCH 052/111] Correction for MDEV-16930. --- sql/sql_yacc.yy | 4 ++-- sql/sql_yacc_ora.yy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 1ec731781a5..7ed44982dd6 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -13345,7 +13345,7 @@ values_with_names: if (unlikely(Lex->insert_list->push_back($4, thd->mem_root))) MYSQL_YYABORT; // give some name in case of using in table value constuctor (TVC) - if (!$4->name.str) + if (!$4->name.str || $4->name.str == item_empty_name) $4->set_name(thd, $3, (uint) ($5 - $3), thd->charset()); } | remember_name expr_or_default remember_end @@ -13353,7 +13353,7 @@ values_with_names: if (unlikely(Lex->insert_list->push_back($2, thd->mem_root))) MYSQL_YYABORT; // give some name in case of using in table value constuctor (TVC) - if (!$2->name.str) + if (!$2->name.str || $2->name.str == item_empty_name) $2->set_name(thd, $1, (uint) ($3 - $1), thd->charset()); } ; diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index a6446180fc0..8f98cfa3694 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -13500,7 +13500,7 @@ values_with_names: if (unlikely(Lex->insert_list->push_back($4, thd->mem_root))) MYSQL_YYABORT; // give some name in case of using in table value constuctor (TVC) - if (!$4->name.str) + if (!$4->name.str || $4->name.str == item_empty_name) $4->set_name(thd, $3, (uint) ($5 - $3), thd->charset()); } | remember_name expr_or_default remember_end @@ -13508,7 +13508,7 @@ values_with_names: if (unlikely(Lex->insert_list->push_back($2, thd->mem_root))) MYSQL_YYABORT; // give some name in case of using in table value constuctor (TVC) - if (!$2->name.str) + if (!$2->name.str || $2->name.str == item_empty_name) $2->set_name(thd, $1, (uint) ($3 - $1), thd->charset()); } ; From db3be33ec877a3e39a18d46da8dc3ac364be8113 Mon Sep 17 00:00:00 2001 From: Nikita Malyavin Date: Tue, 24 Jul 2018 01:59:15 +1000 Subject: [PATCH 053/111] MDEV-16783 Assertion `!conds' failed in mysql_delete upon 2nd execution of SP with DELETE HISTORY * remove assertion * do not setup `conds` if it's already cached Fixes #823 --- mysql-test/suite/versioning/r/truncate.result | 6 ++++++ mysql-test/suite/versioning/t/truncate.test | 10 ++++++++++ sql/sql_delete.cc | 14 +++++++++----- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/versioning/r/truncate.result b/mysql-test/suite/versioning/r/truncate.result index 308501915dc..d38b9eed162 100644 --- a/mysql-test/suite/versioning/r/truncate.result +++ b/mysql-test/suite/versioning/r/truncate.result @@ -100,5 +100,11 @@ ERROR 42S02: 'v' is a view unlock tables; drop view v; drop table t; +create table t1 (i int) with system versioning; +create procedure pr() delete history from t1 before system_time now(); +call pr; +call pr; +drop procedure pr; +drop table t1; drop database test; create database test; diff --git a/mysql-test/suite/versioning/t/truncate.test b/mysql-test/suite/versioning/t/truncate.test index 4e039a76567..1727c7787ef 100644 --- a/mysql-test/suite/versioning/t/truncate.test +++ b/mysql-test/suite/versioning/t/truncate.test @@ -107,5 +107,15 @@ unlock tables; drop view v; drop table t; +# +# MDEV-16783 Assertion `!conds' failed in mysql_delete upon 2nd execution of SP with DELETE HISTORY +# +create table t1 (i int) with system versioning; +create procedure pr() delete history from t1 before system_time now(); +call pr; +call pr; +drop procedure pr; +drop table t1; + drop database test; create database test; diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index b6ca2e956cb..a040ba9eb93 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -323,12 +323,16 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, DBUG_ASSERT(table); DBUG_ASSERT(!conds || thd->stmt_arena->is_stmt_execute()); - if (select_lex->vers_setup_conds(thd, table_list)) - DBUG_RETURN(TRUE); - DBUG_ASSERT(!conds); - conds= table_list->on_expr; - table_list->on_expr= NULL; + // conds could be cached from previous SP call + if (!conds) + { + if (select_lex->vers_setup_conds(thd, table_list)) + DBUG_RETURN(TRUE); + + conds= table_list->on_expr; + table_list->on_expr= NULL; + } } if (mysql_handle_list_of_derived(thd->lex, table_list, DT_MERGE_FOR_INSERT)) From 9c5a0380019ba16016708b4eb9031818830f8fb9 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 23 Aug 2018 12:45:14 +0200 Subject: [PATCH 054/111] MDEV-17040 sql/sql_yacc_orac.cc is included but sql/sql_yacc_ora.cc isn't included in source archive fix a packaging typo --- cmake/make_dist.cmake.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/make_dist.cmake.in b/cmake/make_dist.cmake.in index 43498f1f888..6fad17137fd 100644 --- a/cmake/make_dist.cmake.in +++ b/cmake/make_dist.cmake.in @@ -110,7 +110,7 @@ CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.cc CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.hh ${PACKAGE_DIR}/sql/sql_yacc_ora.hh COPYONLY) CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.cc - ${PACKAGE_DIR}/sql/sql_yacc_orac.cc COPYONLY) + ${PACKAGE_DIR}/sql/sql_yacc_ora.cc COPYONLY) # Add documentation, if user has specified where to find them IF(MYSQL_DOCS_LOCATION) From 4c652fc62e76481b2b608fb35545f6ca1d5987fa Mon Sep 17 00:00:00 2001 From: benrubson Date: Mon, 12 Feb 2018 22:08:57 +0100 Subject: [PATCH 055/111] Use stunnel during rsync SST if available (cherry picked from commit 1adc382c2fb4300059d130521a24a1dab141531e) --- scripts/wsrep_sst_rsync.sh | 74 ++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 88b1352db2e..44625a549e3 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -41,6 +41,8 @@ cleanup_joiner() kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || \ : rm -rf "$RSYNC_CONF" + rm -f "$STUNNEL_CONF" + rm -f "$STUNNEL_PID" rm -rf "$MAGIC_FILE" rm -rf "$RSYNC_PID" wsrep_log_info "Joiner cleanup done." @@ -68,7 +70,7 @@ check_pid_and_port() local port_info="$(sockstat -46lp ${rsync_port} 2>/dev/null | \ grep ":${rsync_port}")" local is_rsync="$(echo $port_info | \ - grep '[[:space:]]\+rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null)" + grep -E '[[:space:]]+(rsync|stunnel)[[:space:]]+'"$rsync_pid" 2>/dev/null)" ;; *) if ! which lsof > /dev/null; then @@ -79,7 +81,7 @@ check_pid_and_port() local port_info="$(lsof -i :$rsync_port -Pn 2>/dev/null | \ grep "(LISTEN)")" local is_rsync="$(echo $port_info | \ - grep -w '^rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null)" + grep -E '^(rsync|stunnel)[[:space:]]+'"$rsync_pid" 2>/dev/null)" ;; esac @@ -119,6 +121,12 @@ is_local_ip() $get_addr_bin | grep "$address" > /dev/null } +STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf" +rm -f "$STUNNEL_CONF" + +STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" +rm -f "$STUNNEL_PID" + MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" rm -rf "$MAGIC_FILE" @@ -156,9 +164,28 @@ fi FILTER="-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*'" +SSTKEY=$(parse_cnf sst tkey "") +SSTCERT=$(parse_cnf sst tcert "") +STUNNEL="" +if [ -f "$SSTKEY" ] && [ -f "$SSTCERT" ] && wsrep_check_programs stunnel +then + STUNNEL="stunnel ${STUNNEL_CONF}" +fi + if [ "$WSREP_SST_OPT_ROLE" = "donor" ] then +cat << EOF > "$STUNNEL_CONF" +CApath = ${SSTCERT%/*} +foreground = yes +pid = $STUNNEL_PID +debug = warning +client = yes +connect = ${WSREP_SST_OPT_ADDR%/*} +TIMEOUTclose = 0 +verifyPeer = yes +EOF + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] then @@ -220,7 +247,8 @@ then # first, the normal directories, so that we can detect incompatible protocol RC=0 - eval rsync --owner --group --perms --links --specials \ + eval rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT ${FILTER} "$WSREP_SST_OPT_DATA/" \ rsync://$WSREP_SST_OPT_ADDR >&2 || RC=$? @@ -243,7 +271,8 @@ then fi # second, we transfer InnoDB log files - rsync --owner --group --perms --links --specials \ + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '- **' "$WSREP_LOG_DIR/" \ rsync://$WSREP_SST_OPT_ADDR-log_dir >&2 || RC=$? @@ -263,7 +292,8 @@ then find . -maxdepth 1 -mindepth 1 -type d -not -name "lost+found" \ -print0 | xargs -I{} -0 -P $count \ - rsync --owner --group --perms --links --specials \ + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ --ignore-times --inplace --recursive --delete --quiet \ $WHOLE_FILE_OPT --exclude '*/ib_logfile*' "$WSREP_SST_OPT_DATA"/{}/ \ rsync://$WSREP_SST_OPT_ADDR/{} >&2 || RC=$? @@ -286,7 +316,8 @@ then echo "continue" # now server can resume updating data echo "$STATE" > "$MAGIC_FILE" - rsync --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR echo "done $STATE" @@ -347,14 +378,37 @@ EOF # If the IP is local listen only in it if is_local_ip "$RSYNC_ADDR" then - rsync --daemon --no-detach --address "$RSYNC_ADDR" --port "$RSYNC_PORT" --config "$RSYNC_CONF" & + RSYNC_EXTRA_ARGS="--address $RSYNC_ADDR" + STUNNEL_ACCEPT="$RSYNC_ADDR:$RSYNC_PORT" else - # Not local, possibly a NAT, listen in all interface - rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" & + # Not local, possibly a NAT, listen on all interfaces + RSYNC_EXTRA_ARGS="" + STUNNEL_ACCEPT="$RSYNC_PORT" # Overwrite address with all RSYNC_ADDR="*" fi - RSYNC_REAL_PID=$! + + if [ -z "$STUNNEL" ] + then + rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" ${RSYNC_EXTRA_ARGS} & + RSYNC_REAL_PID=$! + else + cat << EOF > "$STUNNEL_CONF" +key = $SSTKEY +cert = $SSTCERT +foreground = yes +pid = $STUNNEL_PID +debug = warning +client = no +[rsync] +accept = $STUNNEL_ACCEPT +exec = $(which rsync) +execargs = rsync --server --daemon --config=$RSYNC_CONF . +EOF + stunnel "$STUNNEL_CONF" & + RSYNC_REAL_PID=$! + RSYNC_PID=$STUNNEL_PID + fi until check_pid_and_port "$RSYNC_PID" "$RSYNC_REAL_PID" "$RSYNC_ADDR" "$RSYNC_PORT" do From c826b6b8da3c9ee89ab457155bae96310213e5a1 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 24 Aug 2018 20:53:00 -0700 Subject: [PATCH 056/111] Added a new parameter for the function eq_ranges_exceeds_limit() introduced in the patch fo MDEV-16934. --- sql/multi_range_read.cc | 8 +++++--- sql/opt_range.cc | 5 ++--- sql/opt_range.h | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc index 94a96c27a0f..9b6d0e86f90 100644 --- a/sql/multi_range_read.cc +++ b/sql/multi_range_read.cc @@ -64,13 +64,15 @@ handler::multi_range_read_info_const(uint keyno, RANGE_SEQ_IF *seq, ha_rows rows, total_rows= 0; uint n_ranges=0; THD *thd= table->in_use; + uint limit= thd->variables.eq_range_index_dive_limit; + bool use_statistics_for_eq_range= eq_ranges_exceeds_limit(seq, + seq_init_param, + limit); + /* Default MRR implementation doesn't need buffer */ *bufsz= 0; - bool use_statistics_for_eq_range= eq_ranges_exceeds_limit(seq, - seq_init_param); - seq_it= seq->init(seq_init_param, n_ranges, *flags); while (!seq->next(seq_it, &range)) { diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 47e07016782..2c015c1a5da 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -14619,13 +14619,12 @@ void QUICK_GROUP_MIN_MAX_SELECT::add_keys_and_lengths(String *key_names, /* Check whether the number for equality ranges exceeds the set threshold */ -bool eq_ranges_exceeds_limit(RANGE_SEQ_IF *seq, void *seq_init_param) +bool eq_ranges_exceeds_limit(RANGE_SEQ_IF *seq, void *seq_init_param, + uint limit) { KEY_MULTI_RANGE range; range_seq_t seq_it; uint count = 0; - PARAM *param= ((SEL_ARG_RANGE_SEQ*) seq_init_param)->param; - uint limit= param->thd->variables.eq_range_index_dive_limit; if (limit == 0) { diff --git a/sql/opt_range.h b/sql/opt_range.h index 6698c987e78..9e0bd3ae9ff 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -1664,7 +1664,8 @@ SQL_SELECT *make_select(TABLE *head, table_map const_tables, bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item **cond); -bool eq_ranges_exceeds_limit(RANGE_SEQ_IF *seq, void *seq_init_param); +bool eq_ranges_exceeds_limit(RANGE_SEQ_IF *seq, void *seq_init_param, + uint limit); #ifdef WITH_PARTITION_STORAGE_ENGINE bool prune_partitions(THD *thd, TABLE *table, Item *pprune_cond); From d526679efd108478cc2af07578a15434fb46ed85 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Tue, 3 Apr 2018 12:41:13 +0000 Subject: [PATCH 057/111] MDEV-14474 information_schema.check_constraints Implement according to standard SQL specification 2008. The check_constraints table is used for fetching metadata about the constraints defined for tables in all databases. --- mysql-test/main/information_schema.result | 2 + .../information_schema_all_engines.result | 8 +- .../funcs_1/r/is_check_constraints.result | 144 ++++++++++++++++++ .../suite/funcs_1/r/is_columns_is.result | 10 ++ .../suite/funcs_1/r/is_tables_is.result | 50 ++++++ .../suite/funcs_1/t/is_check_constraints.test | 69 +++++++++ sql/handler.h | 1 + sql/sql_parse.cc | 1 + sql/sql_show.cc | 50 ++++++ 9 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/funcs_1/r/is_check_constraints.result create mode 100644 mysql-test/suite/funcs_1/t/is_check_constraints.test diff --git a/mysql-test/main/information_schema.result b/mysql-test/main/information_schema.result index 0e6b4168ea5..83f987b78db 100644 --- a/mysql-test/main/information_schema.result +++ b/mysql-test/main/information_schema.result @@ -51,6 +51,7 @@ c ALL_PLUGINS APPLICABLE_ROLES CHARACTER_SETS +CHECK_CONSTRAINTS CLIENT_STATISTICS COLLATIONS COLLATION_CHARACTER_SET_APPLICABILITY @@ -938,6 +939,7 @@ connection user10261; SELECT TABLE_NAME, COLUMN_NAME, PRIVILEGES FROM INFORMATION_SCHEMA.COLUMNS where COLUMN_NAME='TABLE_NAME' and table_name not like 'innodb%'; TABLE_NAME COLUMN_NAME PRIVILEGES +CHECK_CONSTRAINTS TABLE_NAME select COLUMNS TABLE_NAME select COLUMN_PRIVILEGES TABLE_NAME select FILES TABLE_NAME select diff --git a/mysql-test/main/information_schema_all_engines.result b/mysql-test/main/information_schema_all_engines.result index 8a92f0226ff..2916858b5a6 100644 --- a/mysql-test/main/information_schema_all_engines.result +++ b/mysql-test/main/information_schema_all_engines.result @@ -4,6 +4,7 @@ Tables_in_information_schema ALL_PLUGINS APPLICABLE_ROLES CHARACTER_SETS +CHECK_CONSTRAINTS CLIENT_STATISTICS COLLATIONS COLLATION_CHARACTER_SET_APPLICABILITY @@ -83,6 +84,7 @@ table_name column_name ALL_PLUGINS PLUGIN_NAME APPLICABLE_ROLES GRANTEE CHARACTER_SETS CHARACTER_SET_NAME +CHECK_CONSTRAINTS CONSTRAINT_SCHEMA CLIENT_STATISTICS CLIENT COLLATIONS COLLATION_NAME COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME @@ -162,6 +164,7 @@ table_name column_name ALL_PLUGINS PLUGIN_NAME APPLICABLE_ROLES GRANTEE CHARACTER_SETS CHARACTER_SET_NAME +CHECK_CONSTRAINTS CONSTRAINT_SCHEMA CLIENT_STATISTICS CLIENT COLLATIONS COLLATION_NAME COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME @@ -247,6 +250,7 @@ table_name group_concat(t.table_schema, '.', t.table_name) num1 ALL_PLUGINS information_schema.ALL_PLUGINS 1 APPLICABLE_ROLES information_schema.APPLICABLE_ROLES 1 CHARACTER_SETS information_schema.CHARACTER_SETS 1 +CHECK_CONSTRAINTS information_schema.CHECK_CONSTRAINTS 1 CLIENT_STATISTICS information_schema.CLIENT_STATISTICS 1 COLLATIONS information_schema.COLLATIONS 1 COLLATION_CHARACTER_SET_APPLICABILITY information_schema.COLLATION_CHARACTER_SET_APPLICABILITY 1 @@ -315,6 +319,7 @@ Database: information_schema | ALL_PLUGINS | | APPLICABLE_ROLES | | CHARACTER_SETS | +| CHECK_CONSTRAINTS | | CLIENT_STATISTICS | | COLLATIONS | | COLLATION_CHARACTER_SET_APPLICABILITY | @@ -384,6 +389,7 @@ Database: INFORMATION_SCHEMA | ALL_PLUGINS | | APPLICABLE_ROLES | | CHARACTER_SETS | +| CHECK_CONSTRAINTS | | CLIENT_STATISTICS | | COLLATIONS | | COLLATION_CHARACTER_SET_APPLICABILITY | @@ -453,5 +459,5 @@ Wildcard: inf_rmation_schema | information_schema | SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') GROUP BY TABLE_SCHEMA; table_schema count(*) -information_schema 64 +information_schema 65 mysql 31 diff --git a/mysql-test/suite/funcs_1/r/is_check_constraints.result b/mysql-test/suite/funcs_1/r/is_check_constraints.result new file mode 100644 index 00000000000..65565d89884 --- /dev/null +++ b/mysql-test/suite/funcs_1/r/is_check_constraints.result @@ -0,0 +1,144 @@ +# +# MDEV-14474: Create INFORMATION_SCHEMA.CHECK_CONSTRAINTS +# +set check_constraint_checks=1; +use test; +create table t0 +( +t int, check (t>32) # table constraint +) ENGINE=myisam; +SELECT * from information_schema.check_constraints; +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CONSTRAINT_1 +TABLE_NAME t0 +CHECK_CLAUSE `t` > 32 +ALTER TABLE t0 +ADD CONSTRAINT CHK_t0_t CHECK(t<100); +SELECT * from information_schema.check_constraints; +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CONSTRAINT_1 +TABLE_NAME t0 +CHECK_CLAUSE `t` > 32 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CHK_t0_t +TABLE_NAME t0 +CHECK_CLAUSE `t` < 100 +ALTER TABLE t0 +DROP CONSTRAINT CHK_t0_t; +SELECT * from information_schema.check_constraints; +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CONSTRAINT_1 +TABLE_NAME t0 +CHECK_CLAUSE `t` > 32 +CREATE TABLE t1 +( t int CHECK(t>2), # field constraint +tt int, CONSTRAINT CHK_tt CHECK(tt<100) # table constraint +) ENGINE=InnoDB; +SELECT * from information_schema.check_constraints; +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME t +TABLE_NAME t1 +CHECK_CLAUSE `t` > 2 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CHK_tt +TABLE_NAME t1 +CHECK_CLAUSE `tt` < 100 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CONSTRAINT_1 +TABLE_NAME t0 +CHECK_CLAUSE `t` > 32 +ALTER TABLE t1 +DROP CONSTRAINT CHK_tt; +SELECT * from information_schema.check_constraints; +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME t +TABLE_NAME t1 +CHECK_CLAUSE `t` > 2 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CONSTRAINT_1 +TABLE_NAME t0 +CHECK_CLAUSE `t` > 32 +create table t2 +( +name VARCHAR(30) CHECK(CHAR_LENGTH(name)>2), #field constraint +start_date DATE, +end_date DATE, +CONSTRAINT CHK_dates CHECK(start_date IS NULL) #table constraint +)ENGINE=Innodb; +SELECT * from information_schema.check_constraints; +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME name +TABLE_NAME t2 +CHECK_CLAUSE char_length(`name`) > 2 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CHK_dates +TABLE_NAME t2 +CHECK_CLAUSE `start_date` is null +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME t +TABLE_NAME t1 +CHECK_CLAUSE `t` > 2 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CONSTRAINT_1 +TABLE_NAME t0 +CHECK_CLAUSE `t` > 32 +ALTER TABLE t1 +ADD CONSTRAINT CHK_new_ CHECK(t>tt); +SELECT * from information_schema.check_constraints; +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME name +TABLE_NAME t2 +CHECK_CLAUSE char_length(`name`) > 2 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CHK_dates +TABLE_NAME t2 +CHECK_CLAUSE `start_date` is null +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME t +TABLE_NAME t1 +CHECK_CLAUSE `t` > 2 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CHK_new_ +TABLE_NAME t1 +CHECK_CLAUSE `t` > `tt` +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CONSTRAINT_1 +TABLE_NAME t0 +CHECK_CLAUSE `t` > 32 +create table t3 +( +a int, +b int check (b>0), # field constraint named 'b' +CONSTRAINT b check (b>10) # table constraint +) ENGINE=InnoDB; +select * from information_schema.check_constraints; +CONSTRAINT_CATALOG CONSTRAINT_SCHEMA CONSTRAINT_NAME TABLE_NAME CHECK_CLAUSE +def test b t3 `b` > 0 +def test b t3 `b` > 10 +def test name t2 char_length(`name`) > 2 +def test CHK_dates t2 `start_date` is null +def test t t1 `t` > 2 +def test CHK_new_ t1 `t` > `tt` +def test CONSTRAINT_1 t0 `t` > 32 +drop table t0; +drop table t1; +drop table t2; +drop table t3; diff --git a/mysql-test/suite/funcs_1/r/is_columns_is.result b/mysql-test/suite/funcs_1/r/is_columns_is.result index d76efef3e08..dfaa5d75137 100644 --- a/mysql-test/suite/funcs_1/r/is_columns_is.result +++ b/mysql-test/suite/funcs_1/r/is_columns_is.result @@ -24,6 +24,11 @@ def information_schema CHARACTER_SETS CHARACTER_SET_NAME 1 '' NO varchar 32 96 N def information_schema CHARACTER_SETS DEFAULT_COLLATE_NAME 2 '' NO varchar 32 96 NULL NULL NULL utf8 utf8_general_ci varchar(32) select NEVER NULL def information_schema CHARACTER_SETS DESCRIPTION 3 '' NO varchar 60 180 NULL NULL NULL utf8 utf8_general_ci varchar(60) select NEVER NULL def information_schema CHARACTER_SETS MAXLEN 4 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(3) select NEVER NULL +def information_schema CHECK_CONSTRAINTS CHECK_CLAUSE 5 '' NO varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select NEVER NULL +def information_schema CHECK_CONSTRAINTS CONSTRAINT_CATALOG 1 '' NO varchar 512 1536 NULL NULL NULL utf8 utf8_general_ci varchar(512) select NEVER NULL +def information_schema CHECK_CONSTRAINTS CONSTRAINT_NAME 3 '' NO varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select NEVER NULL +def information_schema CHECK_CONSTRAINTS CONSTRAINT_SCHEMA 2 '' NO varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select NEVER NULL +def information_schema CHECK_CONSTRAINTS TABLE_NAME 4 '' NO varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select NEVER NULL def information_schema CLIENT_STATISTICS ACCESS_DENIED 22 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(21) select NEVER NULL def information_schema CLIENT_STATISTICS BINLOG_BYTES_WRITTEN 9 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(21) select NEVER NULL def information_schema CLIENT_STATISTICS BUSY_TIME 5 0 NO double NULL NULL 21 NULL NULL NULL NULL double select NEVER NULL @@ -557,6 +562,11 @@ COL_CML TABLE_SCHEMA TABLE_NAME COLUMN_NAME DATA_TYPE CHARACTER_MAXIMUM_LENGTH C 3.0000 information_schema CHARACTER_SETS DEFAULT_COLLATE_NAME varchar 32 96 utf8 utf8_general_ci varchar(32) 3.0000 information_schema CHARACTER_SETS DESCRIPTION varchar 60 180 utf8 utf8_general_ci varchar(60) NULL information_schema CHARACTER_SETS MAXLEN bigint NULL NULL NULL NULL bigint(3) +3.0000 information_schema CHECK_CONSTRAINTS CONSTRAINT_CATALOG varchar 512 1536 utf8 utf8_general_ci varchar(512) +3.0000 information_schema CHECK_CONSTRAINTS CONSTRAINT_SCHEMA varchar 64 192 utf8 utf8_general_ci varchar(64) +3.0000 information_schema CHECK_CONSTRAINTS CONSTRAINT_NAME varchar 64 192 utf8 utf8_general_ci varchar(64) +3.0000 information_schema CHECK_CONSTRAINTS TABLE_NAME varchar 64 192 utf8 utf8_general_ci varchar(64) +3.0000 information_schema CHECK_CONSTRAINTS CHECK_CLAUSE varchar 64 192 utf8 utf8_general_ci varchar(64) 3.0000 information_schema CLIENT_STATISTICS CLIENT varchar 64 192 utf8 utf8_general_ci varchar(64) NULL information_schema CLIENT_STATISTICS TOTAL_CONNECTIONS bigint NULL NULL NULL NULL bigint(21) NULL information_schema CLIENT_STATISTICS CONCURRENT_CONNECTIONS bigint NULL NULL NULL NULL bigint(21) diff --git a/mysql-test/suite/funcs_1/r/is_tables_is.result b/mysql-test/suite/funcs_1/r/is_tables_is.result index e0e5a82a265..5fee1e0050a 100644 --- a/mysql-test/suite/funcs_1/r/is_tables_is.result +++ b/mysql-test/suite/funcs_1/r/is_tables_is.result @@ -89,6 +89,31 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME CHECK_CONSTRAINTS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +MAX_INDEX_LENGTH #MIL# +TEMPORARY Y +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME CLIENT_STATISTICS TABLE_TYPE SYSTEM VIEW ENGINE MEMORY @@ -1105,6 +1130,31 @@ user_comment Separator ----------------------------------------------------- TABLE_CATALOG def TABLE_SCHEMA information_schema +TABLE_NAME CHECK_CONSTRAINTS +TABLE_TYPE SYSTEM VIEW +ENGINE MEMORY +VERSION 11 +ROW_FORMAT Fixed +TABLE_ROWS #TBLR# +AVG_ROW_LENGTH #ARL# +DATA_LENGTH #DL# +MAX_DATA_LENGTH #MDL# +INDEX_LENGTH #IL# +DATA_FREE #DF# +AUTO_INCREMENT NULL +CREATE_TIME #CRT# +UPDATE_TIME #UT# +CHECK_TIME #CT# +TABLE_COLLATION utf8_general_ci +CHECKSUM NULL +CREATE_OPTIONS #CO# +TABLE_COMMENT #TC# +MAX_INDEX_LENGTH #MIL# +TEMPORARY Y +user_comment +Separator ----------------------------------------------------- +TABLE_CATALOG def +TABLE_SCHEMA information_schema TABLE_NAME CLIENT_STATISTICS TABLE_TYPE SYSTEM VIEW ENGINE MEMORY diff --git a/mysql-test/suite/funcs_1/t/is_check_constraints.test b/mysql-test/suite/funcs_1/t/is_check_constraints.test new file mode 100644 index 00000000000..23110ccd20c --- /dev/null +++ b/mysql-test/suite/funcs_1/t/is_check_constraints.test @@ -0,0 +1,69 @@ +--source include/have_innodb.inc +--echo # +--echo # MDEV-14474: Create INFORMATION_SCHEMA.CHECK_CONSTRAINTS +--echo # + +set check_constraint_checks=1; + +use test; +create table t0 +( + t int, check (t>32) # table constraint +) ENGINE=myisam; + +--vertical_results +SELECT * from information_schema.check_constraints; + +ALTER TABLE t0 +ADD CONSTRAINT CHK_t0_t CHECK(t<100); + +SELECT * from information_schema.check_constraints; + +ALTER TABLE t0 +DROP CONSTRAINT CHK_t0_t; + +SELECT * from information_schema.check_constraints; + +CREATE TABLE t1 +( t int CHECK(t>2), # field constraint + tt int, CONSTRAINT CHK_tt CHECK(tt<100) # table constraint +) ENGINE=InnoDB; + +SELECT * from information_schema.check_constraints; + +ALTER TABLE t1 +DROP CONSTRAINT CHK_tt; + +SELECT * from information_schema.check_constraints; + +create table t2 +( +name VARCHAR(30) CHECK(CHAR_LENGTH(name)>2), #field constraint +start_date DATE, +end_date DATE, +CONSTRAINT CHK_dates CHECK(start_date IS NULL) #table constraint +)ENGINE=Innodb; + +SELECT * from information_schema.check_constraints; + +ALTER TABLE t1 +ADD CONSTRAINT CHK_new_ CHECK(t>tt); + +SELECT * from information_schema.check_constraints; + + +# Create table with same field and table check constraint name +create table t3 +( +a int, +b int check (b>0), # field constraint named 'b' +CONSTRAINT b check (b>10) # table constraint +) ENGINE=InnoDB; + +--horizontal_results +select * from information_schema.check_constraints; + +drop table t0; +drop table t1; +drop table t2; +drop table t3; diff --git a/sql/handler.h b/sql/handler.h index c6d7ef5870a..e3ab87b941b 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -950,6 +950,7 @@ enum enum_schema_tables SCH_ALL_PLUGINS, SCH_APPLICABLE_ROLES, SCH_CHARSETS, + SCH_CHECK_CONSTRAINTS, SCH_COLLATIONS, SCH_COLLATION_CHARACTER_SET_APPLICABILITY, SCH_COLUMNS, diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index c1ec9ff94e6..c68e91bbef3 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2565,6 +2565,7 @@ int prepare_schema_table(THD *thd, LEX *lex, Table_ident *table_ident, case SCH_TABLE_NAMES: case SCH_TABLES: + case SCH_CHECK_CONSTRAINTS: case SCH_VIEWS: case SCH_TRIGGERS: case SCH_EVENTS: diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 05a1cce4175..6feaafbe0c7 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -6788,6 +6788,42 @@ store_constraints(THD *thd, TABLE *table, const LEX_CSTRING *db_name, return schema_table_store_record(thd, table); } +static int get_check_constraints_record(THD *thd, TABLE_LIST *tables, + TABLE *table, bool res, + const LEX_CSTRING *db_name, + const LEX_CSTRING *table_name) +{ + DBUG_ENTER("get_check_constraints_record"); + if (res) + { + if (thd->is_error()) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + thd->get_stmt_da()->sql_errno(), + thd->get_stmt_da()->message()); + thd->clear_error(); + DBUG_RETURN(0); + } + else if (!tables->view) + { + if (tables->table->s->table_check_constraints) + { + for (uint i= 0; i < tables->table->s->table_check_constraints; i++) + { + StringBuffer str(system_charset_info); + Virtual_column_info *check= tables->table->check_constraints[i]; + restore_record(table, s->default_values); + table->field[0]->store(STRING_WITH_LEN("def"), system_charset_info); + table->field[1]->store(db_name->str, db_name->length, system_charset_info); + table->field[2]->store(check->name.str, check->name.length, system_charset_info); + table->field[3]->store(table_name->str, table_name->length, system_charset_info); + check->print(&str); + table->field[4]->store(str.ptr(), str.length(), system_charset_info); + schema_table_store_record(thd, table); + } + } + } + DBUG_RETURN(res); +} static int get_schema_constraints_record(THD *thd, TABLE_LIST *tables, TABLE *table, bool res, @@ -9708,6 +9744,18 @@ ST_FIELD_INFO spatial_ref_sys_fields_info[]= #endif /*HAVE_SPATIAL*/ +ST_FIELD_INFO check_constraints_fields_info[]= +{ + {"CONSTRAINT_CATALOG", FN_REFLEN, MYSQL_TYPE_STRING, 0, 0, 0, OPEN_FULL_TABLE}, + {"CONSTRAINT_SCHEMA", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, + OPEN_FULL_TABLE}, + {"CONSTRAINT_NAME", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, + OPEN_FULL_TABLE}, + {"TABLE_NAME", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, OPEN_FULL_TABLE}, + {"CHECK_CLAUSE", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, + OPEN_FULL_TABLE}, + {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} +}; /* Description of ST_FIELD_INFO in table.h @@ -9723,6 +9771,8 @@ ST_SCHEMA_TABLE schema_tables[]= fill_schema_applicable_roles, 0, 0, -1, -1, 0, 0}, {"CHARACTER_SETS", charsets_fields_info, 0, fill_schema_charsets, make_character_sets_old_format, 0, -1, -1, 0, 0}, + {"CHECK_CONSTRAINTS", check_constraints_fields_info, 0, + get_all_tables, 0, get_check_constraints_record, 1, 2, 0, OPTIMIZE_I_S_TABLE|OPEN_TABLE_ONLY}, {"COLLATIONS", collation_fields_info, 0, fill_schema_collation, make_old_format, 0, -1, -1, 0, 0}, {"COLLATION_CHARACTER_SET_APPLICABILITY", coll_charset_app_fields_info, From 2a361ebe1b9c52eadfe811a219a89df6a0d6e3ea Mon Sep 17 00:00:00 2001 From: zhzhzoo Date: Sat, 17 Mar 2018 01:22:53 +0800 Subject: [PATCH 058/111] MDEV-15204: lag/lead function order list mandatory --- mysql-test/r/win.result | 4 ++-- mysql-test/r/win_lead_lag.result | 11 +++++++++++ mysql-test/t/win.test | 4 ++-- mysql-test/t/win_lead_lag.test | 13 +++++++++++++ sql/item_windowfunc.h | 2 ++ 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result index b519b2bb223..e4b2d223a02 100644 --- a/mysql-test/r/win.result +++ b/mysql-test/r/win.result @@ -3219,8 +3219,8 @@ DROP TABLE fv_test, fv_result; # CREATE TABLE t1 (a int); INSERT INTO t1 VALUES (0),(1),(2); -SELECT LEAD(a) OVER (PARTITION BY a) as lead, -a AND LEAD(a) OVER (PARTITION BY a) AS a_and_lead_part +SELECT LEAD(a) OVER (PARTITION BY a ORDER BY a) as lead, +a AND LEAD(a) OVER (PARTITION BY a ORDER BY a) AS a_and_lead_part FROM t1; lead a_and_lead_part NULL 0 diff --git a/mysql-test/r/win_lead_lag.result b/mysql-test/r/win_lead_lag.result index 5ead58fa11a..f3c0b8f7ef2 100644 --- a/mysql-test/r/win_lead_lag.result +++ b/mysql-test/r/win_lead_lag.result @@ -226,4 +226,15 @@ pk a b a+b lag(a + b) over (partition by a order by pk) + pk 9 2 2 4 12 10 2 0 2 14 11 2 10 12 13 +# +# MDEV-15204 - LAG function doesn't require ORDER BY in OVER clause +# +select pk, +lag(pk, 1) over () +from t1; +ERROR HY000: No order list in window specification for 'lag' +select pk, +lead(pk, 1) over () +from t1; +ERROR HY000: No order list in window specification for 'lead' drop table t1; diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test index b354a55d0d6..f47237ee59e 100644 --- a/mysql-test/t/win.test +++ b/mysql-test/t/win.test @@ -2000,8 +2000,8 @@ DROP TABLE fv_test, fv_result; CREATE TABLE t1 (a int); INSERT INTO t1 VALUES (0),(1),(2); -SELECT LEAD(a) OVER (PARTITION BY a) as lead, - a AND LEAD(a) OVER (PARTITION BY a) AS a_and_lead_part +SELECT LEAD(a) OVER (PARTITION BY a ORDER BY a) as lead, + a AND LEAD(a) OVER (PARTITION BY a ORDER BY a) AS a_and_lead_part FROM t1; SELECT a OR LEAD(a) OVER (ORDER BY a) AS a_or_lead_order diff --git a/mysql-test/t/win_lead_lag.test b/mysql-test/t/win_lead_lag.test index 2824f83789c..d154244ecd0 100644 --- a/mysql-test/t/win_lead_lag.test +++ b/mysql-test/t/win_lead_lag.test @@ -107,4 +107,17 @@ select pk, a, b, a+b, from t1 order by pk asc; +--echo # +--echo # MDEV-15204 - LAG function doesn't require ORDER BY in OVER clause +--echo # +--error ER_NO_ORDER_LIST_IN_WINDOW_SPEC +select pk, + lag(pk, 1) over () +from t1; + +--error ER_NO_ORDER_LIST_IN_WINDOW_SPEC +select pk, + lead(pk, 1) over () +from t1; + drop table t1; diff --git a/sql/item_windowfunc.h b/sql/item_windowfunc.h index e5afb9e1555..21270733051 100644 --- a/sql/item_windowfunc.h +++ b/sql/item_windowfunc.h @@ -792,6 +792,8 @@ public: case Item_sum::DENSE_RANK_FUNC: case Item_sum::PERCENT_RANK_FUNC: case Item_sum::CUME_DIST_FUNC: + case Item_sum::LAG_FUNC: + case Item_sum::LEAD_FUNC: return true; default: return false; From b6f055025bd9ba7b533d023d509cb6f755a65db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sat, 25 Aug 2018 19:56:00 +0300 Subject: [PATCH 059/111] Make funcs_1.is_check_constraints deterministic --- .../funcs_1/r/is_check_constraints.result | 50 +++++++++---------- .../suite/funcs_1/t/is_check_constraints.test | 16 +++--- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/mysql-test/suite/funcs_1/r/is_check_constraints.result b/mysql-test/suite/funcs_1/r/is_check_constraints.result index 65565d89884..678cfb8db2f 100644 --- a/mysql-test/suite/funcs_1/r/is_check_constraints.result +++ b/mysql-test/suite/funcs_1/r/is_check_constraints.result @@ -7,7 +7,7 @@ create table t0 ( t int, check (t>32) # table constraint ) ENGINE=myisam; -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test CONSTRAINT_NAME CONSTRAINT_1 @@ -15,20 +15,20 @@ TABLE_NAME t0 CHECK_CLAUSE `t` > 32 ALTER TABLE t0 ADD CONSTRAINT CHK_t0_t CHECK(t<100); -SELECT * from information_schema.check_constraints; -CONSTRAINT_CATALOG def -CONSTRAINT_SCHEMA test -CONSTRAINT_NAME CONSTRAINT_1 -TABLE_NAME t0 -CHECK_CLAUSE `t` > 32 +SELECT * from information_schema.check_constraints order by check_clause; CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test CONSTRAINT_NAME CHK_t0_t TABLE_NAME t0 CHECK_CLAUSE `t` < 100 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CONSTRAINT_1 +TABLE_NAME t0 +CHECK_CLAUSE `t` > 32 ALTER TABLE t0 DROP CONSTRAINT CHK_t0_t; -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test CONSTRAINT_NAME CONSTRAINT_1 @@ -38,12 +38,7 @@ CREATE TABLE t1 ( t int CHECK(t>2), # field constraint tt int, CONSTRAINT CHK_tt CHECK(tt<100) # table constraint ) ENGINE=InnoDB; -SELECT * from information_schema.check_constraints; -CONSTRAINT_CATALOG def -CONSTRAINT_SCHEMA test -CONSTRAINT_NAME t -TABLE_NAME t1 -CHECK_CLAUSE `t` > 2 +SELECT * from information_schema.check_constraints order by check_clause; CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test CONSTRAINT_NAME CHK_tt @@ -51,12 +46,17 @@ TABLE_NAME t1 CHECK_CLAUSE `tt` < 100 CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test +CONSTRAINT_NAME t +TABLE_NAME t1 +CHECK_CLAUSE `t` > 2 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test CONSTRAINT_NAME CONSTRAINT_1 TABLE_NAME t0 CHECK_CLAUSE `t` > 32 ALTER TABLE t1 DROP CONSTRAINT CHK_tt; -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test CONSTRAINT_NAME t @@ -74,7 +74,7 @@ start_date DATE, end_date DATE, CONSTRAINT CHK_dates CHECK(start_date IS NULL) #table constraint )ENGINE=Innodb; -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test CONSTRAINT_NAME name @@ -97,7 +97,7 @@ TABLE_NAME t0 CHECK_CLAUSE `t` > 32 ALTER TABLE t1 ADD CONSTRAINT CHK_new_ CHECK(t>tt); -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test CONSTRAINT_NAME name @@ -115,29 +115,29 @@ TABLE_NAME t1 CHECK_CLAUSE `t` > 2 CONSTRAINT_CATALOG def CONSTRAINT_SCHEMA test -CONSTRAINT_NAME CHK_new_ -TABLE_NAME t1 -CHECK_CLAUSE `t` > `tt` -CONSTRAINT_CATALOG def -CONSTRAINT_SCHEMA test CONSTRAINT_NAME CONSTRAINT_1 TABLE_NAME t0 CHECK_CLAUSE `t` > 32 +CONSTRAINT_CATALOG def +CONSTRAINT_SCHEMA test +CONSTRAINT_NAME CHK_new_ +TABLE_NAME t1 +CHECK_CLAUSE `t` > `tt` create table t3 ( a int, b int check (b>0), # field constraint named 'b' CONSTRAINT b check (b>10) # table constraint ) ENGINE=InnoDB; -select * from information_schema.check_constraints; +select * from information_schema.check_constraints order by check_clause; CONSTRAINT_CATALOG CONSTRAINT_SCHEMA CONSTRAINT_NAME TABLE_NAME CHECK_CLAUSE +def test name t2 char_length(`name`) > 2 def test b t3 `b` > 0 def test b t3 `b` > 10 -def test name t2 char_length(`name`) > 2 def test CHK_dates t2 `start_date` is null def test t t1 `t` > 2 -def test CHK_new_ t1 `t` > `tt` def test CONSTRAINT_1 t0 `t` > 32 +def test CHK_new_ t1 `t` > `tt` drop table t0; drop table t1; drop table t2; diff --git a/mysql-test/suite/funcs_1/t/is_check_constraints.test b/mysql-test/suite/funcs_1/t/is_check_constraints.test index 23110ccd20c..b39abdc1b24 100644 --- a/mysql-test/suite/funcs_1/t/is_check_constraints.test +++ b/mysql-test/suite/funcs_1/t/is_check_constraints.test @@ -12,29 +12,29 @@ create table t0 ) ENGINE=myisam; --vertical_results -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; ALTER TABLE t0 ADD CONSTRAINT CHK_t0_t CHECK(t<100); -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; ALTER TABLE t0 DROP CONSTRAINT CHK_t0_t; -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; CREATE TABLE t1 ( t int CHECK(t>2), # field constraint tt int, CONSTRAINT CHK_tt CHECK(tt<100) # table constraint ) ENGINE=InnoDB; -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; ALTER TABLE t1 DROP CONSTRAINT CHK_tt; -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; create table t2 ( @@ -44,12 +44,12 @@ end_date DATE, CONSTRAINT CHK_dates CHECK(start_date IS NULL) #table constraint )ENGINE=Innodb; -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; ALTER TABLE t1 ADD CONSTRAINT CHK_new_ CHECK(t>tt); -SELECT * from information_schema.check_constraints; +SELECT * from information_schema.check_constraints order by check_clause; # Create table with same field and table check constraint name @@ -61,7 +61,7 @@ CONSTRAINT b check (b>10) # table constraint ) ENGINE=InnoDB; --horizontal_results -select * from information_schema.check_constraints; +select * from information_schema.check_constraints order by check_clause; drop table t0; drop table t1; From 2b76f6f61dba93fd920456a0dd9efd366a6dc6b1 Mon Sep 17 00:00:00 2001 From: Ming Lin Date: Thu, 16 Aug 2018 10:28:29 -0700 Subject: [PATCH 060/111] MDEV-16703: Update AUTO_INCREMENT in the UPDATE statement Currently RocksDB engine doesn't update AUTO_INCREMENT in the UPDATE statement. For example, CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; INSERT INTO t1 (a) VALUES (1); UPDATE t1 SET pk = 3; ==> AUTO_INCREMENT should be updated to 4. Without this fix, it hits the Assertion `dd_val >= last_val' failed in myrocks::ha_rocksdb::load_auto_incr_value_from_index. (cherry picked from commit f7154242b8807fdaa74ed0b89c3de8c35ee35edb) --- storage/rocksdb/ha_rocksdb.cc | 2 +- .../rocksdb/mysql-test/rocksdb/r/autoinc_vars.result | 8 ++++++++ storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test | 10 ++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index c1c3191514f..adef71e4e4a 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -9553,7 +9553,7 @@ int ha_rocksdb::update_pk(const Rdb_key_def &kd, } } - if (table->next_number_field) { + if (table->found_next_number_field) { update_auto_incr_val_from_field(); } diff --git a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result index 8cc9b070a70..0c496227006 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result @@ -150,3 +150,11 @@ CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; INSERT INTO t0 VALUES(0); ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; +#--------------------------------------------------------------- +# MDEV-16703 Assertion failed in load_auto_incr_value_from_index +#--------------------------------------------------------------- +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test index 9d7f0365d07..86a0a7f2a53 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars.test @@ -116,3 +116,13 @@ CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; INSERT INTO t0 VALUES(0); ALTER TABLE t0 AUTO_INCREMENT=0; DROP TABLE t0; + +--echo #--------------------------------------------------------------- +--echo # MDEV-16703 Assertion failed in load_auto_incr_value_from_index +--echo #--------------------------------------------------------------- + +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; From a290b807e8ebba48a079e32cd53f33eca2a7caf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 27 Aug 2018 12:03:02 +0300 Subject: [PATCH 061/111] MDEV-17062: Test failure on galera.MW-336 MDEV-17058: Test failure on wsrep.variables MDEV-17060: Test failure on galera.galera_var_slave_threads Fix incorrect calculation of increased applier (slave) threads. Note that increase change takes effect "immediately" but we should use proper wait condition to wait it. Reducing the number of slave threads is not immediate as thread will only exit after a replication event. --- mysql-test/suite/galera/r/MW-336.result | 100 +++++++++++++++++++----- mysql-test/suite/galera/t/MW-336.test | 65 +++++++++------ mysql-test/suite/wsrep/t/variables.test | 9 ++- sql/wsrep_var.cc | 4 +- 4 files changed, 129 insertions(+), 49 deletions(-) diff --git a/mysql-test/suite/galera/r/MW-336.result b/mysql-test/suite/galera/r/MW-336.result index 0bf8d9d3909..1b5d2984e39 100644 --- a/mysql-test/suite/galera/r/MW-336.result +++ b/mysql-test/suite/galera/r/MW-336.result @@ -2,39 +2,99 @@ CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB; connection node_1; SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 1; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +COUNT(*) +11 +SHOW STATUS LIKE 'wsrep_thread_count'; +Variable_name Value +wsrep_thread_count 11 connection node_2; INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); connection node_1; SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 20; SET GLOBAL wsrep_slave_threads = 1; connection node_2; INSERT INTO t1 VALUES (1); -INSERT INTO t1 VALUES (2); -INSERT INTO t1 VALUES (3); -INSERT INTO t1 VALUES (4); -INSERT INTO t1 VALUES (5); -INSERT INTO t1 VALUES (6); -INSERT INTO t1 VALUES (7); -INSERT INTO t1 VALUES (8); -INSERT INTO t1 VALUES (9); -connection node_1; +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 0; Warnings: Warning 1292 Truncated incorrect wsrep_slave_threads value: '0' connection node_2; -INSERT INTO t1 VALUES (10); -INSERT INTO t1 VALUES (11); -INSERT INTO t1 VALUES (12); -INSERT INTO t1 VALUES (13); -INSERT INTO t1 VALUES (14); -INSERT INTO t1 VALUES (15); -INSERT INTO t1 VALUES (16); -INSERT INTO t1 VALUES (17); -INSERT INTO t1 VALUES (18); -INSERT INTO t1 VALUES (19); -INSERT INTO t1 VALUES (20); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); connection node_1; SET GLOBAL wsrep_slave_threads = 1; +connection node_2; +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (1); +connection node_1; DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MW-336.test b/mysql-test/suite/galera/t/MW-336.test index 8cd363aa019..4bdbfb2ecff 100644 --- a/mysql-test/suite/galera/t/MW-336.test +++ b/mysql-test/suite/galera/t/MW-336.test @@ -8,15 +8,28 @@ CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB; --connection node_1 + SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 1; + +--let $wait_timeout=600 --let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); --source include/wait_condition.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); +SHOW STATUS LIKE 'wsrep_thread_count'; + --connection node_2 -INSERT INTO t1 VALUES (1); +# Generate 11 replication events +--let $count = 11 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} --connection node_1 + SET GLOBAL wsrep_slave_threads = 10; --let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); --source include/wait_condition.inc @@ -28,40 +41,40 @@ SET GLOBAL wsrep_slave_threads = 20; SET GLOBAL wsrep_slave_threads = 1; --connection node_2 -INSERT INTO t1 VALUES (1); -INSERT INTO t1 VALUES (2); -INSERT INTO t1 VALUES (3); -INSERT INTO t1 VALUES (4); -INSERT INTO t1 VALUES (5); -INSERT INTO t1 VALUES (6); -INSERT INTO t1 VALUES (7); -INSERT INTO t1 VALUES (8); -INSERT INTO t1 VALUES (9); - - ---connection node_1 ---let $wait_condition = SELECT COUNT(*) = 12 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); ---source include/wait_condition.inc +# Generate 21 replication events +--let $count = 21 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} SET GLOBAL wsrep_slave_threads = 10; SET GLOBAL wsrep_slave_threads = 0; --connection node_2 -INSERT INTO t1 VALUES (10); -INSERT INTO t1 VALUES (11); -INSERT INTO t1 VALUES (12); -INSERT INTO t1 VALUES (13); -INSERT INTO t1 VALUES (14); -INSERT INTO t1 VALUES (15); -INSERT INTO t1 VALUES (16); -INSERT INTO t1 VALUES (17); -INSERT INTO t1 VALUES (18); -INSERT INTO t1 VALUES (19); -INSERT INTO t1 VALUES (20); +# Generate 21 replication events +--let $count = 21 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} --connection node_1 --let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%'); --source include/wait_condition.inc SET GLOBAL wsrep_slave_threads = 1; + +--connection node_2 +# Generate 21 replication events +--let $count = 21 +while ($count) +{ + INSERT INTO t1 VALUES (1); + --dec $count +} + +--connection node_1 DROP TABLE t1; diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test index 1315f090d5c..40b869422ff 100644 --- a/mysql-test/suite/wsrep/t/variables.test +++ b/mysql-test/suite/wsrep/t/variables.test @@ -101,8 +101,10 @@ SHOW STATUS LIKE 'wsrep_thread_count'; --echo # Setting wsrep_cluster_address triggers the creation of --echo # applier/rollbacker threads. SET GLOBAL wsrep_cluster_address= 'gcomm://'; + --echo # Wait for applier threads to get created. -sleep 3; +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count'; +--source include/wait_condition.inc --replace_regex /.*libgalera_smm.*/libgalera_smm.so/ SELECT @@global.wsrep_provider; @@ -113,8 +115,11 @@ SHOW STATUS LIKE 'wsrep_thread_count'; SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads; SET GLOBAL wsrep_slave_threads= 10; + --echo # Wait for applier threads to get created. -sleep 3; +--let $wait_condition = SELECT VARIABLE_VALUE = 11 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count'; +--source include/wait_condition.inc + SHOW STATUS LIKE 'threads_connected'; SHOW STATUS LIKE 'wsrep_thread_count'; diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 7fe120beb8e..aaae487b2e2 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -594,7 +594,9 @@ void wsrep_node_address_init (const char* value) static void wsrep_slave_count_change_update () { - wsrep_slave_count_change += (wsrep_slave_threads - wsrep_prev_slave_threads); + wsrep_slave_count_change = (wsrep_slave_threads - wsrep_prev_slave_threads); + WSREP_DEBUG("Change on slave threads: New %lu old %lu difference %lu", + wsrep_slave_threads, wsrep_prev_slave_threads, wsrep_slave_count_change); wsrep_prev_slave_threads = wsrep_slave_threads; } From 55163ba1bdb1a05daadc66c41c959994231b361c Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Wed, 1 Aug 2018 20:44:59 +0300 Subject: [PATCH 062/111] MDEV-16803: Pushdown Item_func_in item that uses vectors in several SELECTs The bug appears because of the Item_func_in::build_clone() method. The 'array' field for the Item_func_in item that can be pushed into the materialized view/derived table was built in the wrong way. It becomes lame after the pushdown of the condition into the first SELECT that defines that view/derived table. The server crashes in the pushdown into the next SELECT while trying to use already lame 'array' field. To fix it Item_func_in::build_clone() was changed. --- mysql-test/r/derived_cond_pushdown.result | 21 +++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 20 ++++++++++++++++++++ sql/item_cmpfunc.cc | 3 +-- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index fd930d13547..9ef3488e06d 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -10220,3 +10220,24 @@ EXPLAIN } } DROP TABLE t1; +# +# MDEV-16803: pushdown condition with IN predicate in the derived table +# defined with several SELECT statements +# +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2),(3,2),(1,1); +SELECT * FROM +( +SELECT a,b,1 as c +FROM t1 +UNION ALL +SELECT a,b,2 as c +FROM t1 +) AS tab +WHERE ((a,b) IN ((1,2),(3,2))); +a b c +1 2 1 +3 2 1 +1 2 2 +3 2 2 +DROP TABLE t1; diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index b8e74a2d5aa..23109662c6d 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -2022,3 +2022,23 @@ EVAL $query; EVAL EXPLAIN FORMAT=JSON $query; DROP TABLE t1; + +--echo # +--echo # MDEV-16803: pushdown condition with IN predicate in the derived table +--echo # defined with several SELECT statements +--echo # + +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2),(3,2),(1,1); + +SELECT * FROM +( + SELECT a,b,1 as c + FROM t1 + UNION ALL + SELECT a,b,2 as c + FROM t1 +) AS tab +WHERE ((a,b) IN ((1,2),(3,2))); + +DROP TABLE t1; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 577a98df0d2..69ebeb2b95c 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4436,9 +4436,8 @@ Item *Item_func_in::build_clone(THD *thd, MEM_ROOT *mem_root) Item_func_in *clone= (Item_func_in *) Item_func::build_clone(thd, mem_root); if (clone) { - if (array && clone->create_array(thd)) - return NULL; bzero(&clone->cmp_items, sizeof(cmp_items)); + clone->fix_length_and_dec(); } return clone; } From 497d86276f4b67191d58d491ab294dc06d6f606a Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 27 Aug 2018 08:12:59 -0700 Subject: [PATCH 063/111] MDEV-17017 Explain for query using derived table specified with a table value constructor shows wrong number of rows This is another attempt to fix this bug. The previous patch did not take into account that a transformation for ALL/ANY subqueries could be applied to the materialized table that wrapped the table value constructor used as a specification of the subselect used an ALL/ANY subquery. In this case the result of the derived table used a sink of the class select_subselect rather than of the class select_unit. Thus the previous fix could cause memory overwrites when running EXPLAIN for queries with table value constructors in ALL/ANY subselects. --- mysql-test/main/table_value_constr.test | 2 -- sql/sql_class.h | 5 ++--- sql/sql_lex.cc | 6 +++--- sql/table.cc | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/mysql-test/main/table_value_constr.test b/mysql-test/main/table_value_constr.test index 5df40d1047c..0dd0a7a04b0 100644 --- a/mysql-test/main/table_value_constr.test +++ b/mysql-test/main/table_value_constr.test @@ -1123,5 +1123,3 @@ PREPARE stmt FROM "SELECT * FROM (VALUES(1 + 1,2,'abc')) t"; EXECUTE stmt; EXECUTE stmt; DEALLOCATE PREPARE stmt; - - diff --git a/sql/sql_class.h b/sql/sql_class.h index faae443e7a0..026ef4d1118 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4956,7 +4956,8 @@ protected: SELECT_LEX_UNIT *unit; /* Something used only by the parser: */ public: - select_result(THD *thd_arg): select_result_sink(thd_arg) {} + ha_rows est_records; /* estimated number of records in the result */ + select_result(THD *thd_arg): select_result_sink(thd_arg), est_records(0) {} void set_unit(SELECT_LEX_UNIT *unit_arg) { unit= unit_arg; } virtual ~select_result() {}; /** @@ -5528,7 +5529,6 @@ public: TMP_TABLE_PARAM tmp_table_param; int write_err; /* Error code from the last send_data->ha_write_row call. */ TABLE *table; - ha_rows records; select_unit(THD *thd_arg): select_result_interceptor(thd_arg), @@ -5566,7 +5566,6 @@ public: curr_sel= UINT_MAX; step= UNION_TYPE; write_err= 0; - records= 0; } void change_select(); }; diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 7d88a6cab4b..1b1031d440b 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -4678,18 +4678,18 @@ void SELECT_LEX::increase_derived_records(ha_rows records) return; } - select_unit *result= (select_unit*)unit->result; + select_result *result= unit->result; switch (linkage) { case INTERSECT_TYPE: // result of intersect can't be more then one of components - set_if_smaller(result->records, records); + set_if_smaller(result->est_records, records); case EXCEPT_TYPE: // in worse case none of record will be removed break; default: // usual UNION - result->records+= records; + result->est_records+= records; break; } } diff --git a/sql/table.cc b/sql/table.cc index 627f6159979..4aef9aad4fc 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -8334,7 +8334,7 @@ int TABLE_LIST::fetch_number_of_rows() } if (is_materialized_derived() && !fill_me) { - table->file->stats.records= ((select_unit*)(get_unit()->result))->records; + table->file->stats.records= get_unit()->result->est_records; set_if_bigger(table->file->stats.records, 2); table->used_stat_records= table->file->stats.records; } From cded083a370f4b23c6d895d44f2948c267ed5e77 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 27 Aug 2018 22:00:14 +0300 Subject: [PATCH 064/111] MDEV-15797 Assertion `thd->killed != 0' failed in ha_maria::enable_indexes Problem was that a parallel open of a table, overwrote info->state that was in used by repair. Fixed by changing _ma_tmp_disable_logging_for_table() to use a new state buffer state.no_logging to store the temporary state. Other things: - Use original number of rows when retrying repair to get rid of a potential warning "Number of rows changed from X to Y" - Changed maria_commit() to make it easier to merge with 10.4 - If table is not locked (like with show commands), use the global number of rows as the local number may not be up to date. (Minor not critical fix) - Added some missing DBUG_RETURN --- mysql-test/suite/maria/concurrent.result | 28 ++++++++++++++++++++++++ mysql-test/suite/maria/concurrent.test | 28 ++++++++++++++++++++++++ storage/maria/ha_maria.cc | 3 +++ storage/maria/ma_commit.c | 2 +- storage/maria/ma_info.c | 6 ++++- storage/maria/ma_recovery.c | 8 +++++-- storage/maria/maria_chk.c | 4 ++-- storage/maria/maria_def.h | 2 ++ 8 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/maria/concurrent.result create mode 100644 mysql-test/suite/maria/concurrent.test diff --git a/mysql-test/suite/maria/concurrent.result b/mysql-test/suite/maria/concurrent.result new file mode 100644 index 00000000000..5498a6410a2 --- /dev/null +++ b/mysql-test/suite/maria/concurrent.result @@ -0,0 +1,28 @@ +CREATE TABLE t1 (a INT, b CHAR(12), c INT, FULLTEXT KEY(b), KEY (c)) ENGINE=Aria; +CREATE TABLE t2 (a INT, b CHAR(12), c INT) ENGINE=Aria; +INSERT INTO t2 VALUES (1,'foo',8), (2,'bar',9); +INSERT INTO t1 SELECT * FROM t2; +select 1; +1 +1 +select 1; +1 +1 +select 1; +1 +1 +select 1; +1 +1 +select 1; +1 +1 +select 1; +1 +1 +select 1; +1 +1 +SELECT * FROM t1 WHERE a = ( SELECT 1 FROM non_existing_table2 ); +ERROR 42S02: Table 'test.non_existing_table2' doesn't exist +DROP TABLE t1, t2; diff --git a/mysql-test/suite/maria/concurrent.test b/mysql-test/suite/maria/concurrent.test new file mode 100644 index 00000000000..42adb082d40 --- /dev/null +++ b/mysql-test/suite/maria/concurrent.test @@ -0,0 +1,28 @@ +# +# MDEV-15797 Assertion `thd->killed != 0' failed in ha_maria::enable_indexes +# + +CREATE TABLE t1 (a INT, b CHAR(12), c INT, FULLTEXT KEY(b), KEY (c)) ENGINE=Aria; +CREATE TABLE t2 (a INT, b CHAR(12), c INT) ENGINE=Aria; +INSERT INTO t2 VALUES (1,'foo',8), (2,'bar',9); + +--connect (con1,localhost,root,,test) +--send + INSERT INTO t1 SELECT * FROM t2; +--connection default +select 1; +select 1; +select 1; +select 1; +select 1; +select 1; +select 1; +--error ER_NO_SUCH_TABLE +SELECT * FROM t1 WHERE a = ( SELECT 1 FROM non_existing_table2 ); +--connection con1 +--reap + +# Cleanup +--disconnect con1 +--connection default +DROP TABLE t1, t2; diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index eee6b502c47..d438aa90378 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -1460,6 +1460,7 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) while ((error= repair(thd, param, 0)) && param->retry_repair) { param->retry_repair= 0; + file->state->records= start_records; if (test_all_bits(param->testflag, (uint) (T_RETRY_WITHOUT_QUICK | T_QUICK))) { @@ -1961,6 +1962,7 @@ int ha_maria::disable_indexes(uint mode) int ha_maria::enable_indexes(uint mode) { int error; + ha_rows start_rows= file->state->records; DBUG_PRINT("info", ("ha_maria::enable_indexes mode: %d", mode)); if (maria_is_all_keys_active(file->s->state.key_map, file->s->base.keys)) { @@ -2023,6 +2025,7 @@ int ha_maria::enable_indexes(uint mode) DBUG_ASSERT(thd->killed != 0); /* Repairing by sort failed. Now try standard repair method. */ param->testflag &= ~T_REP_BY_SORT; + file->state->records= start_rows; error= (repair(thd, param, 0) != HA_ADMIN_OK); /* If the standard repair succeeded, clear all error messages which diff --git a/storage/maria/ma_commit.c b/storage/maria/ma_commit.c index 0ae3868dbf6..3850af8246c 100644 --- a/storage/maria/ma_commit.c +++ b/storage/maria/ma_commit.c @@ -98,7 +98,7 @@ int maria_commit(MARIA_HA *info) if (!info->s->now_transactional) return 0; trn= info->trn; - info->trn= 0; /* checked in maria_close() */ + _ma_reset_trn_for_table(info); return ma_commit(trn); } diff --git a/storage/maria/ma_info.c b/storage/maria/ma_info.c index 1189594fd2b..da44da123d2 100644 --- a/storage/maria/ma_info.c +++ b/storage/maria/ma_info.c @@ -56,7 +56,11 @@ int maria_status(MARIA_HA *info, register MARIA_INFO *x, uint flag) } if (flag & HA_STATUS_VARIABLE) { - x->records = info->state->records; + /* If table is locked, give versioned number otherwise last commited */ + if (info->lock_type == F_UNLCK) + x->records = share->state.state.records; + else + x->records = info->state->records; x->deleted = share->state.state.del; x->delete_length = share->state.state.empty; x->data_file_length = share->state.state.data_file_length; diff --git a/storage/maria/ma_recovery.c b/storage/maria/ma_recovery.c index c06e46c104f..f023dde7817 100644 --- a/storage/maria/ma_recovery.c +++ b/storage/maria/ma_recovery.c @@ -3551,8 +3551,8 @@ void _ma_tmp_disable_logging_for_table(MARIA_HA *info, info->state may point to a state that was deleted by _ma_trnman_end_trans_hook() */ - share->state.common= *info->state; - info->state= &share->state.common; + share->state.no_logging= *info->state; + info->state= &share->state.no_logging; info->switched_transactional= TRUE; /* @@ -3608,6 +3608,10 @@ my_bool _ma_reenable_logging_for_table(MARIA_HA *info, my_bool flush_pages) _ma_copy_nontrans_state_information(info); _ma_reset_history(info->s); + /* Reset state to point to state.common, as on open() */ + info->state= &share->state.common; + *info->state= share->state.state; + if (flush_pages) { /* Ensure that recover is not executing any redo before this */ diff --git a/storage/maria/maria_chk.c b/storage/maria/maria_chk.c index b47f1b8c824..d03c50891df 100644 --- a/storage/maria/maria_chk.c +++ b/storage/maria/maria_chk.c @@ -1128,7 +1128,7 @@ static int maria_chk(HA_CHECK *param, char *filename) { fprintf(stderr, "Aria table '%s' is not fixed because of errors\n", filename); - return(-1); + DBUG_RETURN(-1); } recreate=1; if (!(param->testflag & T_REP_ANY)) @@ -1150,7 +1150,7 @@ static int maria_chk(HA_CHECK *param, char *filename) param->total_deleted+=info->state->del; descript(param, info, filename); maria_close(info); /* Should always succeed */ - return(0); + DBUG_RETURN(0); } if (!stopwords_inited++) diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index 8262e1b5e2d..128e78da32b 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -149,6 +149,8 @@ typedef struct st_maria_state_info MARIA_STATUS_INFO state; /* maria_ha->state points here for crash-safe but not versioned tables */ MARIA_STATUS_INFO common; + /* State for a versioned table that is temporary non versioned */ + MARIA_STATUS_INFO no_logging; ha_rows split; /* number of split blocks */ my_off_t dellink; /* Link to next removed block */ pgcache_page_no_t first_bitmap_with_space; From faa4d8f8c6c12c09541c4733773ac700c5e643cf Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 28 Aug 2018 08:23:44 +0000 Subject: [PATCH 065/111] Copy of commit de1e8c7bfe7c875ea284b55040e8f3cd3a56fcc2 Author: Abhinav Sharma Date: Thu Aug 23 14:34:39 2018 -0700 Log updates to semi-sync whitelist in the error log Summary: Plugin variable changes are not logged in the error log even when log_global_var_changes is enabled. Logging updates to whitelist will help in debugging. Reviewed By: guokeno0 Differential Revision: D9483807 fbshipit-source-id: e111cda773d --- storage/rocksdb/CMakeLists.txt | 23 +- storage/rocksdb/ha_rocksdb.cc | 1329 +++++++++++++---- storage/rocksdb/ha_rocksdb.h | 47 +- storage/rocksdb/ha_rocksdb_proto.h | 7 +- .../rocksdb/mysql-test/rocksdb/combinations | 6 + .../rocksdb/include/have_write_committed.inc | 3 + .../rocksdb/r/add_index_inplace.result | 24 + .../r/add_index_inplace_sstfilewriter.result | 2 +- .../rocksdb/r/allow_no_primary_key.result | 25 + .../mysql-test/rocksdb/r/autoinc_debug.result | 40 +- .../mysql-test/rocksdb/r/autoinc_vars.result | 18 + .../mysql-test/rocksdb/r/bloomfilter5.result | 62 + .../rocksdb/r/bloomfilter_bulk_load.result | 15 + .../mysql-test/rocksdb/r/bulk_load_sk.result | 229 +++ .../mysql-test/rocksdb/r/cardinality.result | 17 +- .../mysql-test/rocksdb/r/collation.result | 20 +- .../mysql-test/rocksdb/r/com_rpc_tx.result | 21 + .../r/create_no_primary_key_table.result | 38 + .../rocksdb/r/ddl_high_priority.result | 71 +- .../rocksdb/r/deadlock_tracking.result | 57 + .../mysql-test/rocksdb/r/drop_table.result | 11 + .../rocksdb/r/explicit_snapshot.result | 265 ++++ .../mysql-test/rocksdb/r/i_s_deadlock.result | 55 +- .../rocksdb/r/index_file_map.result | 3 + .../rocksdb/r/index_merge_rocksdb2.result | 78 +- .../rocksdb/r/information_schema.result | 1 + .../mysql-test/rocksdb/r/issue255.result | 2 +- .../rocksdb/mysql-test/rocksdb/r/lock.result | 15 + .../mysql-test/rocksdb/r/mysqldump.result | 5 +- .../percona_nonflushing_analyze_debug.result | 19 + .../r/prefix_extractor_override.result | 7 +- .../mysql-test/rocksdb/r/read_only_tx.result | 2 +- .../mysql-test/rocksdb/r/rocksdb.result | 24 +- .../rocksdb/r/rocksdb_checksums.result | 6 +- .../r/rocksdb_deadlock_detect_rc.result | 16 +- .../r/rocksdb_deadlock_detect_rr.result | 16 +- .../r/secondary_key_update_lock.result | 11 + .../mysql-test/rocksdb/r/show_engine.result | 53 + .../mysql-test/rocksdb/r/transaction.result | 41 + .../mysql-test/rocksdb/r/trx_info_rpl.result | 2 +- .../mysql-test/rocksdb/r/update.result | 8 + .../rocksdb/r/use_direct_reads.result | 18 + .../rocksdb/r/validate_datadic.result | 3 - .../rocksdb/t/add_index_inplace.test | 18 + .../rocksdb/t/add_index_inplace_crash.test | 1 + .../t/add_index_inplace_sstfilewriter.test | 2 +- .../rocksdb/t/allow_no_primary_key.test | 28 + .../mysql-test/rocksdb/t/autoinc_debug.test | 17 +- .../mysql-test/rocksdb/t/autoinc_vars.test | 23 + .../rocksdb/t/bloomfilter5-master.opt | 3 + .../mysql-test/rocksdb/t/bloomfilter5.test | 61 + .../t/bloomfilter_bulk_load-master.opt | 2 + .../rocksdb/t/bloomfilter_bulk_load.test | 35 + .../mysql-test/rocksdb/t/bulk_load_sk.test | 119 ++ .../mysql-test/rocksdb/t/cardinality.test | 21 +- .../t/check_ignore_unknown_options.test | 4 +- .../mysql-test/rocksdb/t/collation.test | 42 +- .../mysql-test/rocksdb/t/com_rpc_tx.cnf | 4 + .../mysql-test/rocksdb/t/com_rpc_tx.test | 87 ++ .../t/create_no_primary_key_table-master.opt | 1 + .../t/create_no_primary_key_table.test | 44 + .../rocksdb/t/deadlock_tracking.test | 76 +- .../mysql-test/rocksdb/t/drop_table.test | 27 + .../rocksdb/t/explicit_snapshot-master.opt | 1 + .../rocksdb/t/explicit_snapshot.test | 263 ++++ .../mysql-test/rocksdb/t/i_s_deadlock.test | 14 +- .../mysql-test/rocksdb/t/index_file_map.test | 3 + .../rocksdb/t/index_merge_rocksdb2-master.opt | 2 +- .../rocksdb/t/information_schema.test | 1 + .../t/insert_optimized_config-master.opt | 8 - .../rocksdb/t/insert_optimized_config.test | 10 + .../mysql-test/rocksdb/t/issue255.test | 2 +- .../rocksdb/mysql-test/rocksdb/t/lock.test | 22 + .../t/percona_nonflushing_analyze_debug.test | 11 + .../rocksdb/t/prefix_extractor_override.test | 12 +- .../rocksdb/mysql-test/rocksdb/t/rocksdb.test | 3 +- .../rocksdb/t/rocksdb_cf_options.test | 1 - .../rocksdb/t/rocksdb_checksums.test | 24 +- .../rocksdb/t/rocksdb_deadlock_detect.inc | 35 +- .../rocksdb/t/secondary_key_update_lock.test | 26 + .../mysql-test/rocksdb/t/set_checkpoint.inc | 2 +- .../mysql-test/rocksdb/t/show_engine.test | 13 + .../mysql-test/rocksdb/t/transaction.test | 53 + .../mysql-test/rocksdb/t/trx_info_rpl.test | 4 +- .../rocksdb/mysql-test/rocksdb/t/update.test | 10 + .../rocksdb/t/use_direct_reads.test | 37 + .../rocksdb/t/validate_datadic.test | 27 +- .../mysql-test/rocksdb_rpl/combinations | 8 +- .../r/rpl_missing_columns_sk_update.result | 62 + ...mts_dependency_unique_key_conflicts.result | 44 + .../rocksdb_rpl/r/rpl_rocksdb_snapshot.result | 4 +- .../t/rpl_gtid_crash_safe_wal_corrupt.inc | 2 +- .../t/rpl_missing_columns_sk_update.cnf | 13 + .../t/rpl_missing_columns_sk_update.test | 69 + ...l_mts_dependency_unique_key_conflicts.test | 64 + .../t/rpl_rocksdb_2pc_crash_recover.test | 1 + .../mysql-test/rocksdb_stress/combinations | 5 + .../r/rocksdb_block_cache_size_basic.result | 84 +- .../r/rocksdb_bulk_load_allow_sk_basic.result | 100 ++ ...ommit_time_batch_for_recovery_basic.result | 121 ++ .../r/rocksdb_create_checkpoint_basic.result | 2 +- ...debug_manual_compaction_delay_basic.result | 46 + ...error_on_suboptimal_collation_basic.result | 7 + ...sdb_manual_compaction_threads_basic.result | 93 ++ ...ocksdb_max_manifest_file_size_basic.result | 2 +- ...ocksdb_max_manual_compactions_basic.result | 57 + .../r/rocksdb_max_row_locks_basic.result | 21 + .../r/rocksdb_stats_recalc_rate_basic.result | 53 + ...rocksdb_write_batch_max_bytes_basic.result | 4 +- .../r/rocksdb_write_policy_basic.result | 15 + .../t/rocksdb_block_cache_size_basic.test | 16 +- .../t/rocksdb_bulk_load_allow_sk_basic.test | 18 + ..._commit_time_batch_for_recovery_basic.test | 20 + .../t/rocksdb_create_checkpoint_basic.test | 2 +- ...b_debug_manual_compaction_delay_basic.test | 16 + ...b_error_on_suboptimal_collation_basic.test | 6 + ...cksdb_manual_compaction_threads_basic.test | 17 + .../rocksdb_max_manual_compactions_basic.test | 17 + .../t/rocksdb_max_row_locks_basic.test | 1 + .../t/rocksdb_stats_recalc_rate_basic.test | 17 + .../rocksdb_write_batch_max_bytes_basic.test | 4 +- .../t/rocksdb_write_policy_basic.test | 17 + storage/rocksdb/properties_collector.cc | 61 +- storage/rocksdb/properties_collector.h | 3 +- storage/rocksdb/rdb_cf_options.cc | 2 - storage/rocksdb/rdb_comparator.h | 46 +- storage/rocksdb/rdb_datadic.cc | 9 +- storage/rocksdb/rdb_datadic.h | 31 +- storage/rocksdb/rdb_i_s.cc | 167 ++- storage/rocksdb/rdb_i_s.h | 1 + storage/rocksdb/rdb_index_merge.cc | 5 + storage/rocksdb/rdb_perf_context.cc | 2 - storage/rocksdb/rdb_psi.cc | 17 +- storage/rocksdb/rdb_psi.h | 10 +- storage/rocksdb/rdb_sst_info.cc | 4 +- storage/rocksdb/rdb_threads.h | 31 +- storage/rocksdb/rdb_utils.cc | 23 - 137 files changed, 4671 insertions(+), 683 deletions(-) create mode 100644 storage/rocksdb/mysql-test/rocksdb/combinations create mode 100644 storage/rocksdb/mysql-test/rocksdb/include/have_write_committed.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/bloomfilter5.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/bloomfilter_bulk_load.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/bulk_load_sk.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/com_rpc_tx.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/create_no_primary_key_table.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/explicit_snapshot.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/percona_nonflushing_analyze_debug.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/secondary_key_update_lock.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/use_direct_reads.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/bloomfilter5-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/bloomfilter5.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/bloomfilter_bulk_load-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/bloomfilter_bulk_load.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/bulk_load_sk.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/com_rpc_tx.cnf create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/com_rpc_tx.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/create_no_primary_key_table-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/create_no_primary_key_table.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/explicit_snapshot-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/explicit_snapshot.test delete mode 100644 storage/rocksdb/mysql-test/rocksdb/t/insert_optimized_config-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/percona_nonflushing_analyze_debug.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/secondary_key_update_lock.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/use_direct_reads.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_missing_columns_sk_update.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_mts_dependency_unique_key_conflicts.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_missing_columns_sk_update.cnf create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_missing_columns_sk_update.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_mts_dependency_unique_key_conflicts.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_stress/combinations create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_bulk_load_allow_sk_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_commit_time_batch_for_recovery_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_debug_manual_compaction_delay_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_error_on_suboptimal_collation_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_manual_compaction_threads_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_manual_compactions_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_stats_recalc_rate_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_write_policy_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_bulk_load_allow_sk_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_commit_time_batch_for_recovery_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_debug_manual_compaction_delay_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_error_on_suboptimal_collation_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_manual_compaction_threads_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_manual_compactions_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_stats_recalc_rate_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_write_policy_basic.test diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index a5ef5651e1f..54b0b1fab6a 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -82,32 +82,43 @@ ENDIF() SET(rocksdb_static_libs ) IF (WITH_SNAPPY) + FIND_LIBRARY(SNAPPY_LIBRARY + NAMES libsnappy${PIC_EXT}.a snappy + HINTS ${WITH_SNAPPY}/lib) SET(rocksdb_static_libs ${rocksdb_static_libs} - ${WITH_SNAPPY}/lib/libsnappy${PIC_EXT}.a) + ${SNAPPY_LIBRARY}) ADD_DEFINITIONS(-DSNAPPY) ENDIF() IF (WITH_LZ4) + FIND_LIBRARY(LZ4_LIBRARY + NAMES liblz4${PIC_EXT}.a lz4 + HINTS ${WITH_LZ4}/lib) SET(rocksdb_static_libs ${rocksdb_static_libs} - ${WITH_LZ4}/lib/liblz4${PIC_EXT}.a) + ${LZ4_LIBRARY}) ADD_DEFINITIONS(-DLZ4) ENDIF() IF (WITH_BZ2) + FIND_LIBRARY(BZ2_LIBRARY + NAMES libbz2${PIC_EXT}.a bz2 + HINTS ${WITH_BZ2}/lib) SET(rocksdb_static_libs ${rocksdb_static_libs} - ${WITH_BZ2}/lib/libbz2${PIC_EXT}.a) + ${BZ2_LIBRARY}) ADD_DEFINITIONS(-DBZIP2) ENDIF() IF (WITH_ZSTD) - SET(rocksdb_static_libs ${rocksdb_static_libs} - ${WITH_ZSTD}/lib/libzstd${PIC_EXT}.a) + SET(rocksdb_static_libs ${rocksdb_static_libs} ${ZSTD_LIBRARY}) ADD_DEFINITIONS(-DZSTD) ENDIF() IF (WITH_TBB) + FIND_LIBRARY(TBB_LIBRARY + NAMES libtbb${PIC_EXT}.a tbb + HINTS ${WITH_TBB}/lib) SET(rocksdb_static_libs ${rocksdb_static_libs} - ${WITH_TBB}/lib/libtbb${PIC_EXT}.a) + ${TBB_LIBRARY}) ADD_DEFINITIONS(-DTBB) ENDIF() diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 961ca5d1d17..c7d7cc0bdc4 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -25,6 +25,7 @@ /* C++ standard header files */ #include +#include #include #include #include @@ -109,6 +110,58 @@ const std::string DEFAULT_CF_NAME("default"); const std::string DEFAULT_SYSTEM_CF_NAME("__system__"); const std::string PER_INDEX_CF_NAME("$per_index_cf"); +class Rdb_explicit_snapshot; + +std::mutex explicit_snapshot_mutex; +ulonglong explicit_snapshot_counter = 0; +std::unordered_map> + explicit_snapshots; +static std::vector rdb_indexes_to_recalc; + +class Rdb_explicit_snapshot : public explicit_snapshot { + std::unique_ptr snapshot; + + public: + static std::shared_ptr + create(snapshot_info_st *ss_info, rocksdb::DB *db, + const rocksdb::Snapshot *snapshot) { + std::lock_guard lock(explicit_snapshot_mutex); + auto s = std::unique_ptr( + new rocksdb::ManagedSnapshot(db, snapshot)); + if (!s) { + return nullptr; + } + ss_info->snapshot_id = ++explicit_snapshot_counter; + auto ret = std::make_shared(*ss_info, std::move(s)); + if (!ret) { + return nullptr; + } + explicit_snapshots[ss_info->snapshot_id] = ret; + return ret; + } + + static std::shared_ptr + get(const ulonglong snapshot_id) { + std::lock_guard lock(explicit_snapshot_mutex); + auto elem = explicit_snapshots.find(snapshot_id); + if (elem == explicit_snapshots.end()) { + return nullptr; + } + return elem->second.lock(); + } + + rocksdb::ManagedSnapshot *get_snapshot() { return snapshot.get(); } + + Rdb_explicit_snapshot(snapshot_info_st ss_info, + std::unique_ptr snapshot) + : explicit_snapshot(ss_info), snapshot(std::move(snapshot)) {} + + virtual ~Rdb_explicit_snapshot() { + std::lock_guard lock(explicit_snapshot_mutex); + explicit_snapshots.erase(ss_info.snapshot_id); + } +}; + /** Updates row counters based on the table type and operation type. */ @@ -126,11 +179,15 @@ static handler *rocksdb_create_handler(my_core::handlerton *hton, my_core::TABLE_SHARE *table_arg, my_core::MEM_ROOT *mem_root); -static rocksdb::CompactRangeOptions getCompactRangeOptions() { +static rocksdb::CompactRangeOptions +getCompactRangeOptions(int concurrency = 0) { rocksdb::CompactRangeOptions compact_range_options; compact_range_options.bottommost_level_compaction = rocksdb::BottommostLevelCompaction::kForce; compact_range_options.exclusive_manual_compaction = false; + if (concurrency > 0) { + compact_range_options.max_subcompactions = concurrency; + } return compact_range_options; } @@ -167,6 +224,8 @@ Rdb_io_watchdog *io_watchdog = nullptr; static Rdb_background_thread rdb_bg_thread; +static Rdb_manual_compaction_thread rdb_mc_thread; + // List of table names (using regex) that are exceptions to the strict // collation check requirement. Regex_list_handler *rdb_collation_exceptions; @@ -180,30 +239,6 @@ static void rocksdb_flush_all_memtables() { } } -static void rocksdb_compact_column_family_stub( - THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr, - const void *const save) {} - -static int rocksdb_compact_column_family(THD *const thd, - struct st_mysql_sys_var *const var, - void *const var_ptr, - struct st_mysql_value *const value) { - char buff[STRING_BUFFER_USUAL_SIZE]; - int len = sizeof(buff); - - DBUG_ASSERT(value != nullptr); - - if (const char *const cf = value->val_str(value, buff, &len)) { - auto cfh = cf_manager.get_cf(cf); - if (cfh != nullptr && rdb != nullptr) { - sql_print_information("RocksDB: Manual compaction of column family: %s\n", - cf); - rdb->CompactRange(getCompactRangeOptions(), cfh, nullptr, nullptr); - } - } - return HA_EXIT_SUCCESS; -} - /////////////////////////////////////////////////////////// // Hash map: table name => open table handler /////////////////////////////////////////////////////////// @@ -353,6 +388,7 @@ static void rocksdb_drop_index_wakeup_thread( static my_bool rocksdb_pause_background_work = 0; static mysql_mutex_t rdb_sysvars_mutex; +static mysql_mutex_t rdb_block_cache_resize_mutex; static void rocksdb_set_pause_background_work( my_core::THD *const thd MY_ATTRIBUTE((__unused__)), @@ -435,6 +471,9 @@ static void rocksdb_set_wal_bytes_per_sync(THD *thd, struct st_mysql_sys_var *const var, void *const var_ptr, const void *const save); +static int rocksdb_validate_set_block_cache_size( + THD *thd, struct st_mysql_sys_var *const var, void *var_ptr, + struct st_mysql_value *value); ////////////////////////////////////////////////////////////////////////////// // Options definitions ////////////////////////////////////////////////////////////////////////////// @@ -490,11 +529,19 @@ static my_bool rocksdb_enable_bulk_load_api = 1; static my_bool rocksdb_print_snapshot_conflict_queries = 0; static my_bool rocksdb_large_prefix = 0; static my_bool rocksdb_allow_to_start_after_corruption = 0; +static uint64_t rocksdb_write_policy = + rocksdb::TxnDBWritePolicy::WRITE_COMMITTED; +static my_bool rocksdb_error_on_suboptimal_collation = 1; +static uint32_t rocksdb_stats_recalc_rate = 0; +static uint32_t rocksdb_debug_manual_compaction_delay = 0; +static uint32_t rocksdb_max_manual_compactions = 0; std::atomic rocksdb_row_lock_deadlocks(0); std::atomic rocksdb_row_lock_wait_timeouts(0); std::atomic rocksdb_snapshot_conflict_errors(0); std::atomic rocksdb_wal_group_syncs(0); +std::atomic rocksdb_manual_compactions_processed(0); +std::atomic rocksdb_manual_compactions_running(0); static std::unique_ptr rdb_init_rocksdb_db_options(void) { auto o = std::unique_ptr(new rocksdb::DBOptions()); @@ -519,6 +566,14 @@ static std::unique_ptr rocksdb_db_options = static std::shared_ptr rocksdb_rate_limiter; +/* This enum needs to be kept up to date with rocksdb::TxnDBWritePolicy */ +static const char *write_policy_names[] = {"write_committed", "write_prepared", + "write_unprepared", NullS}; + +static TYPELIB write_policy_typelib = {array_elements(write_policy_names) - 1, + "write_policy_typelib", + write_policy_names, nullptr}; + /* This enum needs to be kept up to date with rocksdb::InfoLogLevel */ static const char *info_log_level_names[] = {"debug_level", "info_level", "warn_level", "error_level", @@ -610,6 +665,14 @@ static int rocksdb_validate_flush_log_at_trx_commit( *static_cast(var_ptr) = static_cast(new_value); return HA_EXIT_SUCCESS; } +static void rocksdb_compact_column_family_stub( + THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr, + const void *const save) {} + +static int rocksdb_compact_column_family(THD *const thd, + struct st_mysql_sys_var *const var, + void *const var_ptr, + struct st_mysql_value *const value); static const char *index_type_names[] = {"kBinarySearch", "kHashSearch", NullS}; @@ -618,7 +681,8 @@ static TYPELIB index_type_typelib = {array_elements(index_type_names) - 1, nullptr}; const ulong RDB_MAX_LOCK_WAIT_SECONDS = 1024 * 1024 * 1024; -const ulong RDB_MAX_ROW_LOCKS = 1024 * 1024; +const ulong RDB_DEFAULT_MAX_ROW_LOCKS = 1024 * 1024; +const ulong RDB_MAX_ROW_LOCKS = 1024 * 1024 * 1024; const ulong RDB_DEFAULT_BULK_LOAD_SIZE = 1000; const ulong RDB_MAX_BULK_LOAD_SIZE = 1024 * 1024 * 1024; const size_t RDB_DEFAULT_MERGE_BUF_SIZE = 64 * 1024 * 1024; @@ -649,6 +713,11 @@ static MYSQL_THDVAR_ULONG(deadlock_detect_depth, PLUGIN_VAR_RQCMDARG, /*min*/ 2, /*max*/ ULONG_MAX, 0); +static MYSQL_THDVAR_BOOL( + commit_time_batch_for_recovery, PLUGIN_VAR_RQCMDARG, + "TransactionOptions::commit_time_batch_for_recovery for RocksDB", nullptr, + nullptr, FALSE); + static MYSQL_THDVAR_BOOL( trace_sst_api, PLUGIN_VAR_RQCMDARG, "Generate trace output in the log for each call to the SstFileWriter", @@ -660,6 +729,13 @@ static MYSQL_THDVAR_BOOL( "unique_checks and enables rocksdb_commit_in_the_middle.", rocksdb_check_bulk_load, nullptr, FALSE); +static MYSQL_THDVAR_BOOL(bulk_load_allow_sk, PLUGIN_VAR_RQCMDARG, + "Allow bulk loading of sk keys during bulk-load. " + "Can be changed only when bulk load is disabled.", + /* Intentionally reuse unsorted's check function */ + rocksdb_check_bulk_load_allow_unsorted, nullptr, + FALSE); + static MYSQL_THDVAR_BOOL(bulk_load_allow_unsorted, PLUGIN_VAR_RQCMDARG, "Allow unsorted input during bulk-load. " "Can be changed only when bulk load is disabled.", @@ -705,7 +781,7 @@ static MYSQL_THDVAR_BOOL(skip_bloom_filter_on_read, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ULONG(max_row_locks, PLUGIN_VAR_RQCMDARG, "Maximum number of locks a transaction can have", nullptr, nullptr, - /*default*/ RDB_MAX_ROW_LOCKS, + /*default*/ RDB_DEFAULT_MAX_ROW_LOCKS, /*min*/ 1, /*max*/ RDB_MAX_ROW_LOCKS, 0); @@ -757,6 +833,12 @@ static MYSQL_THDVAR_ULONGLONG( /* min (0ms) */ RDB_MIN_MERGE_TMP_FILE_REMOVAL_DELAY, /* max */ SIZE_T_MAX, 1); +static MYSQL_THDVAR_INT( + manual_compaction_threads, PLUGIN_VAR_RQCMDARG, + "How many rocksdb threads to run for manual compactions", nullptr, nullptr, + /* default rocksdb.dboption max_subcompactions */ 0, + /* min */ 0, /* max */ 128, 0); + static MYSQL_SYSVAR_BOOL( create_if_missing, *reinterpret_cast(&rocksdb_db_options->create_if_missing), @@ -778,6 +860,12 @@ static MYSQL_SYSVAR_BOOL( "DBOptions::manual_wal_flush for RocksDB", nullptr, nullptr, rocksdb_db_options->manual_wal_flush); +static MYSQL_SYSVAR_ENUM(write_policy, rocksdb_write_policy, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "DBOptions::write_policy for RocksDB", nullptr, + nullptr, rocksdb::TxnDBWritePolicy::WRITE_COMMITTED, + &write_policy_typelib); + static MYSQL_SYSVAR_BOOL( create_missing_column_families, *reinterpret_cast( @@ -988,7 +1076,9 @@ static MYSQL_SYSVAR_INT(table_cache_numshardbits, "DBOptions::table_cache_numshardbits for RocksDB", nullptr, nullptr, rocksdb_db_options->table_cache_numshardbits, - /* min */ 0, /* max */ INT_MAX, 0); + // LRUCache limits this to 19 bits, anything greater + // fails to create a cache and returns a nullptr + /* min */ 0, /* max */ 19, 0); static MYSQL_SYSVAR_ULONG(wal_ttl_seconds, rocksdb_db_options->WAL_ttl_seconds, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, @@ -1099,8 +1189,9 @@ static MYSQL_SYSVAR_BOOL( "DBOptions::enable_thread_tracking for RocksDB", nullptr, nullptr, true); static MYSQL_SYSVAR_LONGLONG(block_cache_size, rocksdb_block_cache_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "block_cache size for RocksDB", nullptr, nullptr, + PLUGIN_VAR_RQCMDARG, + "block_cache size for RocksDB", + rocksdb_validate_set_block_cache_size, nullptr, /* default */ RDB_DEFAULT_BLOCK_CACHE_SIZE, /* min */ RDB_MIN_BLOCK_CACHE_SIZE, /* max */ LONGLONG_MAX, @@ -1342,6 +1433,18 @@ static MYSQL_SYSVAR_BOOL( "on PK TTL data. This variable is a no-op in non-debug builds.", nullptr, nullptr, FALSE); +static MYSQL_SYSVAR_UINT( + max_manual_compactions, rocksdb_max_manual_compactions, PLUGIN_VAR_RQCMDARG, + "Maximum number of pending + ongoing number of manual compactions.", + nullptr, nullptr, /* default */ 10, /* min */ 0, /* max */ UINT_MAX, 0); + +static MYSQL_SYSVAR_UINT( + debug_manual_compaction_delay, rocksdb_debug_manual_compaction_delay, + PLUGIN_VAR_RQCMDARG, + "For debugging purposes only. Sleeping specified seconds " + "for simulating long running compactions.", + nullptr, nullptr, 0, /* min */ 0, /* max */ UINT_MAX, 0); + static MYSQL_SYSVAR_BOOL( reset_stats, rocksdb_reset_stats, PLUGIN_VAR_RQCMDARG, "Reset the RocksDB internal statistics without restarting the DB.", nullptr, @@ -1497,6 +1600,13 @@ static MYSQL_SYSVAR_UINT( RDB_DEFAULT_TBL_STATS_SAMPLE_PCT, /* everything */ 0, /* max */ RDB_TBL_STATS_SAMPLE_PCT_MAX, 0); +static MYSQL_SYSVAR_UINT( + stats_recalc_rate, rocksdb_stats_recalc_rate, PLUGIN_VAR_RQCMDARG, + "The number of indexes per second to recalculate statistics for. 0 to " + "disable background recalculation.", + nullptr, nullptr, 0 /* default value */, 0 /* min value */, + UINT_MAX /* max value */, 0); + static MYSQL_SYSVAR_BOOL( large_prefix, rocksdb_large_prefix, PLUGIN_VAR_RQCMDARG, "Support large index prefix length of 3072 bytes. If off, the maximum " @@ -1510,16 +1620,25 @@ static MYSQL_SYSVAR_BOOL( "detected.", nullptr, nullptr, FALSE); +static MYSQL_SYSVAR_BOOL(error_on_suboptimal_collation, + rocksdb_error_on_suboptimal_collation, + PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, + "Raise an error instead of warning if a sub-optimal " + "collation is used", + nullptr, nullptr, TRUE); + static const int ROCKSDB_ASSUMED_KEY_VALUE_DISK_SIZE = 100; static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(lock_wait_timeout), MYSQL_SYSVAR(deadlock_detect), MYSQL_SYSVAR(deadlock_detect_depth), + MYSQL_SYSVAR(commit_time_batch_for_recovery), MYSQL_SYSVAR(max_row_locks), MYSQL_SYSVAR(write_batch_max_bytes), MYSQL_SYSVAR(lock_scanned_rows), MYSQL_SYSVAR(bulk_load), + MYSQL_SYSVAR(bulk_load_allow_sk), MYSQL_SYSVAR(bulk_load_allow_unsorted), MYSQL_SYSVAR(skip_unique_check_tables), MYSQL_SYSVAR(trace_sst_api), @@ -1537,6 +1656,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(create_if_missing), MYSQL_SYSVAR(two_write_queues), MYSQL_SYSVAR(manual_wal_flush), + MYSQL_SYSVAR(write_policy), MYSQL_SYSVAR(create_missing_column_families), MYSQL_SYSVAR(error_if_exists), MYSQL_SYSVAR(paranoid_checks), @@ -1652,6 +1772,11 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(large_prefix), MYSQL_SYSVAR(allow_to_start_after_corruption), + MYSQL_SYSVAR(error_on_suboptimal_collation), + MYSQL_SYSVAR(stats_recalc_rate), + MYSQL_SYSVAR(debug_manual_compaction_delay), + MYSQL_SYSVAR(max_manual_compactions), + MYSQL_SYSVAR(manual_compaction_threads), nullptr}; static rocksdb::WriteOptions @@ -1666,6 +1791,50 @@ rdb_get_rocksdb_write_options(my_core::THD *const thd) { return opt; } +static int rocksdb_compact_column_family(THD *const thd, + struct st_mysql_sys_var *const var, + void *const var_ptr, + struct st_mysql_value *const value) { + char buff[STRING_BUFFER_USUAL_SIZE]; + int len = sizeof(buff); + + DBUG_ASSERT(value != nullptr); + + if (const char *const cf = value->val_str(value, buff, &len)) { + auto cfh = cf_manager.get_cf(cf); + if (cfh != nullptr && rdb != nullptr) { + int mc_id = rdb_mc_thread.request_manual_compaction( + cfh, nullptr, nullptr, THDVAR(thd, manual_compaction_threads)); + if (mc_id == -1) { + my_error(ER_INTERNAL_ERROR, MYF(0), + "Can't schedule more manual compactions. " + "Increase rocksdb_max_manual_compactions or stop issuing " + "more manual compactions."); + return HA_EXIT_FAILURE; + } else if (mc_id < 0) { + return HA_EXIT_FAILURE; + } + // NO_LINT_DEBUG + sql_print_information("RocksDB: Manual compaction of column family: %s\n", + cf); + // Checking thd state every short cycle (100ms). This is for allowing to + // exiting this function without waiting for CompactRange to finish. + do { + my_sleep(100000); + } while (!thd->killed && + !rdb_mc_thread.is_manual_compaction_finished(mc_id)); + + if (thd->killed) { + // This cancels if requested compaction state is INITED. + // TODO(yoshinorim): Cancel running compaction as well once + // it is supported in RocksDB. + rdb_mc_thread.clear_manual_compaction_request(mc_id, true); + } + } + } + return HA_EXIT_SUCCESS; +} + /////////////////////////////////////////////////////////////////////////////////////////// /** @@ -1763,9 +1932,16 @@ protected: bool m_is_delayed_snapshot = false; bool m_is_two_phase = false; - THD *m_thd = nullptr; +private: + /* + Number of write operations this transaction had when we took the last + savepoint (the idea is not to take another savepoint if we haven't made + any changes) + */ + ulonglong m_writes_at_last_savepoint; - rocksdb::ReadOptions m_read_opts; +protected: + THD *m_thd = nullptr; static std::multiset s_tx_list; static mysql_mutex_t s_tx_list_mutex; @@ -1814,7 +1990,16 @@ protected: return s; } +protected: + /* + The following two are helper functions to be overloaded by child classes. + They should provide RocksDB's savepoint semantics. + */ + virtual void do_set_savepoint() = 0; + virtual void do_rollback_to_savepoint() = 0; + public: + rocksdb::ReadOptions m_read_opts; const char *m_mysql_log_file_name; my_off_t m_mysql_log_offset; const char *m_mysql_gtid; @@ -1822,6 +2007,7 @@ protected: String m_detailed_error; int64_t m_snapshot_timestamp = 0; bool m_ddl_transaction; + std::shared_ptr m_explicit_snapshot; /* Tracks the number of tables in use through external_lock. @@ -1881,7 +2067,7 @@ protected: if (s.IsDeadlock()) { my_core::thd_mark_transaction_to_rollback(thd, - false /* just statement */); + true /* whole transaction */); m_detailed_error = String(); table_handler->m_deadlock_counter.inc(); rocksdb_row_lock_deadlocks++; @@ -1898,7 +2084,7 @@ protected: } m_detailed_error = String(" (snapshot conflict)", system_charset_info); table_handler->m_deadlock_counter.inc(); - return HA_ERR_LOCK_DEADLOCK; + return HA_ERR_ROCKSDB_STATUS_BUSY; } if (s.IsIOError() || s.IsCorruption()) { @@ -2298,6 +2484,52 @@ public: virtual bool is_tx_started() const = 0; virtual void start_tx() = 0; virtual void start_stmt() = 0; + + void set_initial_savepoint() { + /* + Set the initial savepoint. If the first statement in the transaction + fails, we need something to roll back to, without rolling back the + entire transaction. + */ + do_set_savepoint(); + m_writes_at_last_savepoint= m_write_count; + } + + /* + Called when a "top-level" statement inside a transaction completes + successfully and its changes become part of the transaction's changes. + */ + void make_stmt_savepoint_permanent() { + + // Take another RocksDB savepoint only if we had changes since the last + // one. This is very important for long transactions doing lots of + // SELECTs. + if (m_writes_at_last_savepoint != m_write_count) + { + do_set_savepoint(); + m_writes_at_last_savepoint= m_write_count; + } + } + + + /* + Rollback to the savepoint we've set before the last statement + */ + void rollback_to_stmt_savepoint() { + if (m_writes_at_last_savepoint != m_write_count) { + do_rollback_to_savepoint(); + /* + RollbackToSavePoint "removes the most recent SetSavePoint()", so + we need to set it again so that next statement can roll back to this + stage. + It's ok to do it here at statement end (instead of doing it at next + statement start) because setting a savepoint is cheap. + */ + do_set_savepoint(); + m_writes_at_last_savepoint= m_write_count; + } + } + virtual void rollback_stmt() = 0; void set_tx_failed(bool failed_arg) { m_is_tx_failed = failed_arg; } @@ -2465,7 +2697,15 @@ public: void acquire_snapshot(bool acquire_now) override { if (m_read_opts.snapshot == nullptr) { - if (is_tx_read_only()) { + const auto thd_ss = std::static_pointer_cast( + m_thd->get_explicit_snapshot()); + if (thd_ss) { + m_explicit_snapshot = thd_ss; + } + if (m_explicit_snapshot) { + auto snapshot = m_explicit_snapshot->get_snapshot()->snapshot(); + snapshot_created(snapshot); + } else if (is_tx_read_only()) { snapshot_created(rdb->GetSnapshot()); } else if (acquire_now) { m_rocksdb_tx->SetSnapshot(); @@ -2482,7 +2722,10 @@ public: if (m_read_opts.snapshot != nullptr) { m_snapshot_timestamp = 0; - if (is_tx_read_only()) { + if (m_explicit_snapshot) { + m_explicit_snapshot.reset(); + need_clear = false; + } else if (is_tx_read_only()) { rdb->ReleaseSnapshot(m_read_opts.snapshot); need_clear = false; } else { @@ -2591,6 +2834,10 @@ public: tx_opts.lock_timeout = rdb_convert_sec_to_ms(m_timeout_sec); tx_opts.deadlock_detect = THDVAR(m_thd, deadlock_detect); tx_opts.deadlock_detect_depth = THDVAR(m_thd, deadlock_detect_depth); + // If this variable is set, this will write commit time write batch + // information on recovery or memtable flush. + tx_opts.use_only_the_last_commit_time_batch_for_recovery = + THDVAR(m_thd, commit_time_batch_for_recovery); tx_opts.max_write_batch_size = THDVAR(m_thd, write_batch_max_bytes); write_opts.sync = (rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC); @@ -2609,9 +2856,20 @@ public: m_read_opts = rocksdb::ReadOptions(); + set_initial_savepoint(); + m_ddl_transaction = false; } + /* Implementations of do_*savepoint based on rocksdB::Transaction savepoints */ + void do_set_savepoint() override { + m_rocksdb_tx->SetSavePoint(); + } + + void do_rollback_to_savepoint() override { + m_rocksdb_tx->RollbackToSavePoint(); + } + /* Start a statement inside a multi-statement transaction. @@ -2624,7 +2882,6 @@ public: void start_stmt() override { // Set the snapshot to delayed acquisition (SetSnapshotOnNextOperation) acquire_snapshot(false); - m_rocksdb_tx->SetSavePoint(); } /* @@ -2635,7 +2892,7 @@ public: /* TODO: here we must release the locks taken since the start_stmt() call */ if (m_rocksdb_tx) { const rocksdb::Snapshot *const org_snapshot = m_rocksdb_tx->GetSnapshot(); - m_rocksdb_tx->RollbackToSavePoint(); + rollback_to_stmt_savepoint(); const rocksdb::Snapshot *const cur_snapshot = m_rocksdb_tx->GetSnapshot(); if (org_snapshot != cur_snapshot) { @@ -2673,7 +2930,7 @@ public: /* This is a rocksdb write batch. This class doesn't hold or wait on any transaction locks (skips rocksdb transaction API) thus giving better - performance. The commit is done through rdb->GetBaseDB()->Commit(). + performance. Currently this is only used for replication threads which are guaranteed to be non-conflicting. Any further usage of this class should completely @@ -2695,6 +2952,8 @@ private: bool commit_no_binlog() override { bool res = false; rocksdb::Status s; + rocksdb::TransactionDBWriteOptimizations optimize; + optimize.skip_concurrency_control = true; s = merge_auto_incr_map(m_batch->GetWriteBatch()); if (!s.ok()) { @@ -2705,7 +2964,7 @@ private: release_snapshot(); - s = rdb->GetBaseDB()->Write(write_opts, m_batch->GetWriteBatch()); + s = rdb->Write(write_opts, optimize, m_batch->GetWriteBatch()); if (!s.ok()) { rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); res = true; @@ -2723,6 +2982,15 @@ error: return res; } + /* Implementations of do_*savepoint based on rocksdB::WriteBatch savepoints */ + void do_set_savepoint() override { + m_batch->SetSavePoint(); + } + + void do_rollback_to_savepoint() override { + m_batch->RollbackToSavePoint(); + } + public: bool is_writebatch_trx() const override { return true; } @@ -2810,6 +3078,13 @@ public: get_for_update(rocksdb::ColumnFamilyHandle *const column_family, const rocksdb::Slice &key, rocksdb::PinnableSlice *const value, bool exclusive) override { + if (value == nullptr) { + rocksdb::PinnableSlice pin_val; + rocksdb::Status s = get(column_family, key, &pin_val); + pin_val.Reset(); + return s; + } + return get(column_family, key, value); } @@ -2828,13 +3103,15 @@ public: write_opts.disableWAL = THDVAR(m_thd, write_disable_wal); write_opts.ignore_missing_column_families = THDVAR(m_thd, write_ignore_missing_column_families); + + set_initial_savepoint(); } - void start_stmt() override { m_batch->SetSavePoint(); } + void start_stmt() override {} void rollback_stmt() override { if (m_batch) - m_batch->RollbackToSavePoint(); + rollback_to_stmt_savepoint(); } explicit Rdb_writebatch_impl(THD *const thd) @@ -3043,6 +3320,8 @@ static int rocksdb_prepare(handlerton *const hton, THD *const thd, DEBUG_SYNC(thd, "rocksdb.prepared"); } + else + tx->make_stmt_savepoint_permanent(); return HA_EXIT_SUCCESS; } @@ -3214,11 +3493,9 @@ static int rocksdb_commit(handlerton *const hton, THD *const thd, } else { /* We get here when committing a statement within a transaction. - - We don't need to do anything here. tx->start_stmt() will notify - Rdb_transaction_impl that another statement has started. */ tx->set_tx_failed(false); + tx->make_stmt_savepoint_permanent(); } if (my_core::thd_tx_isolation(thd) <= ISO_READ_COMMITTED) { @@ -3406,6 +3683,7 @@ private: if (!path_entry.path.empty() && !path_entry.limit_exceeded) { auto deadlocking_txn = *(path_entry.path.end() - 1); deadlock_info.victim_trx_id = deadlocking_txn.m_txn_id; + deadlock_info.deadlock_time = path_entry.deadlock_time; } return deadlock_info; } @@ -3452,16 +3730,18 @@ private: path_data += "\n*** DEADLOCK PATH\n" "=========================================\n"; const auto dl_info = get_dl_path_trx_info(path_entry); + const auto deadlock_time = dl_info.deadlock_time; for (auto it = dl_info.path.begin(); it != dl_info.path.end(); it++) { const auto trx_info = *it; path_data += format_string( + "TIMESTAMP: %" PRId64 "\n" "TRANSACTION ID: %u\n" "COLUMN FAMILY NAME: %s\n" "WAITING KEY: %s\n" "LOCK TYPE: %s\n" "INDEX NAME: %s\n" "TABLE NAME: %s\n", - trx_info.trx_id, trx_info.cf_name.c_str(), + deadlock_time, trx_info.trx_id, trx_info.cf_name.c_str(), trx_info.waiting_key.c_str(), trx_info.exclusive_lock ? "EXCLUSIVE" : "SHARED", trx_info.index_name.c_str(), trx_info.table_name.c_str()); @@ -3772,6 +4052,24 @@ static bool rocksdb_show_status(handlerton *const hton, THD *const thd, str, stat_print); } } + + /* Explicit snapshot information */ + str.clear(); + { + std::lock_guard lock(explicit_snapshot_mutex); + for (const auto &elem : explicit_snapshots) { + const auto &ss = elem.second.lock(); + DBUG_ASSERT(ss != nullptr); + const auto &info = ss->ss_info; + str += "\nSnapshot ID: " + std::to_string(info.snapshot_id) + + "\nBinlog File: " + info.binlog_file + + "\nBinlog Pos: " + std::to_string(info.binlog_pos) + + "\nGtid Executed: " + info.gtid_executed + "\n"; + } + } + if (!str.empty()) { + res |= print_stats(thd, "EXPLICIT_SNAPSHOTS", "rocksdb", str, stat_print); + } } else if (stat_type == HA_ENGINE_TRX) { /* Handle the SHOW ENGINE ROCKSDB TRANSACTION STATUS command */ res |= rocksdb_show_snapshot_status(hton, thd, stat_print); @@ -3791,6 +4089,48 @@ static inline void rocksdb_register_tx(handlerton *const hton, THD *const thd, } } +static bool rocksdb_explicit_snapshot( + handlerton *const /* hton */, /*!< in: RocksDB handlerton */ + THD *const thd, /*!< in: MySQL thread handle */ + snapshot_info_st *ss_info) /*!< out: Snapshot information */ +{ + switch (ss_info->op) { + case snapshot_operation::SNAPSHOT_CREATE: { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } + auto s = Rdb_explicit_snapshot::create(ss_info, rdb, rdb->GetSnapshot()); + if (mysql_bin_log_is_open()) { + mysql_bin_log_unlock_commits(ss_info); + } + + thd->set_explicit_snapshot(s); + return s == nullptr; + } + case snapshot_operation::SNAPSHOT_ATTACH: { + auto s = Rdb_explicit_snapshot::get(ss_info->snapshot_id); + if (!s) { + return true; + } + *ss_info = s->ss_info; + thd->set_explicit_snapshot(s); + return false; + } + case snapshot_operation::SNAPSHOT_RELEASE: { + if (!thd->get_explicit_snapshot()) { + return true; + } + *ss_info = thd->get_explicit_snapshot()->ss_info; + thd->set_explicit_snapshot(nullptr); + return false; + } + default: + DBUG_ASSERT(false); + return true; + } + return true; +} + /* Supporting START TRANSACTION WITH CONSISTENT [ROCKSDB] SNAPSHOT @@ -3813,14 +4153,12 @@ static inline void rocksdb_register_tx(handlerton *const hton, THD *const thd, InnoDB and RocksDB transactions. */ static int rocksdb_start_tx_and_assign_read_view( - handlerton *const hton, /*!< in: RocksDB handlerton */ - THD *const thd, /*!< in: MySQL thread handle of the - user for whom the transaction should - be committed */ - char *const binlog_file, /* out: binlog file for last commit */ - ulonglong *const binlog_pos, /* out: binlog pos for last commit */ - char **gtid_executed, /* out: Gtids logged until last commit */ - int *const gtid_executed_length) /*out: Length of gtid_executed string */ + handlerton *const hton, /*!< in: RocksDB handlerton */ + THD *const thd, /*!< in: MySQL thread handle of the + user for whom the transaction should + be committed */ + snapshot_info_st *ss_info) /*!< in/out: Snapshot info like binlog file, pos, + gtid executed and snapshot ID */ { ulong const tx_isolation = my_core::thd_tx_isolation(thd); @@ -3829,11 +4167,12 @@ static int rocksdb_start_tx_and_assign_read_view( return HA_EXIT_FAILURE; } - if (binlog_file) { - if (binlog_pos && mysql_bin_log_is_open()) - mysql_bin_log_lock_commits(); - else + if (ss_info) { + if (mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } else { return HA_EXIT_FAILURE; + } } Rdb_transaction *const tx = get_or_create_tx(thd); @@ -3844,13 +4183,94 @@ static int rocksdb_start_tx_and_assign_read_view( rocksdb_register_tx(hton, thd, tx); tx->acquire_snapshot(true); - if (binlog_file) - mysql_bin_log_unlock_commits(binlog_file, binlog_pos, gtid_executed, - gtid_executed_length); + if (ss_info) { + mysql_bin_log_unlock_commits(ss_info); + } return HA_EXIT_SUCCESS; } +static int rocksdb_start_tx_with_shared_read_view( + handlerton *const hton, /*!< in: RocksDB handlerton */ + THD *const thd, /*!< in: MySQL thread handle of the + user for whom the transaction should + be committed */ + snapshot_info_st *ss_info) /*!< out: Snapshot info like binlog file, pos, + gtid executed and snapshot ID */ +{ + DBUG_ASSERT(thd != nullptr); + DBUG_ASSERT(ss_info != nullptr); + + int error = HA_EXIT_SUCCESS; + + ulong const tx_isolation = my_core::thd_tx_isolation(thd); + if (tx_isolation != ISO_REPEATABLE_READ) { + my_error(ER_ISOLATION_LEVEL_WITH_CONSISTENT_SNAPSHOT, MYF(0)); + return HA_EXIT_FAILURE; + } + + std::shared_ptr explicit_snapshot; + const auto op = ss_info->op; + Rdb_transaction *tx = nullptr; + + DBUG_ASSERT(op == snapshot_operation::SNAPSHOT_CREATE || + op == snapshot_operation::SNAPSHOT_ATTACH); + + // case: if binlogs are available get binlog file/pos and gtid info + if (op == snapshot_operation::SNAPSHOT_CREATE && mysql_bin_log_is_open()) { + mysql_bin_log_lock_commits(ss_info); + } + + if (op == snapshot_operation::SNAPSHOT_ATTACH) { + explicit_snapshot = Rdb_explicit_snapshot::get(ss_info->snapshot_id); + if (!explicit_snapshot) { + my_printf_error(ER_UNKNOWN_ERROR, "Snapshot %llu does not exist", MYF(0), + ss_info->snapshot_id); + error = HA_EXIT_FAILURE; + } + } + + // case: all good till now + if (error == HA_EXIT_SUCCESS) { + tx = get_or_create_tx(thd); + Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); + + if (explicit_snapshot) { + tx->m_explicit_snapshot = explicit_snapshot; + } + + DBUG_ASSERT(!tx->has_snapshot()); + tx->set_tx_read_only(true); + rocksdb_register_tx(hton, thd, tx); + tx->acquire_snapshot(true); + + // case: an explicit snapshot was not assigned to this transaction + if (!tx->m_explicit_snapshot) { + tx->m_explicit_snapshot = + Rdb_explicit_snapshot::create(ss_info, rdb, tx->m_read_opts.snapshot); + if (!tx->m_explicit_snapshot) { + my_printf_error(ER_UNKNOWN_ERROR, "Could not create snapshot", MYF(0)); + error = HA_EXIT_FAILURE; + } + } + } + + // case: unlock the binlog + if (op == snapshot_operation::SNAPSHOT_CREATE && mysql_bin_log_is_open()) { + mysql_bin_log_unlock_commits(ss_info); + } + + DBUG_ASSERT(error == HA_EXIT_FAILURE || tx->m_explicit_snapshot); + + // copy over the snapshot details to pass to the upper layers + if (tx->m_explicit_snapshot) { + *ss_info = tx->m_explicit_snapshot->ss_info; + ss_info->op = op; + } + + return error; +} + /* Dummy SAVEPOINT support. This is needed for long running transactions * like mysqldump (https://bugs.mysql.com/bug.php?id=71017). * Current SAVEPOINT does not correctly handle ROLLBACK and does not return @@ -4062,9 +4482,11 @@ static int rocksdb_init_func(void *const p) { rdb_bg_thread.init(rdb_signal_bg_psi_mutex_key, rdb_signal_bg_psi_cond_key); rdb_drop_idx_thread.init(rdb_signal_drop_idx_psi_mutex_key, rdb_signal_drop_idx_psi_cond_key); + rdb_mc_thread.init(rdb_signal_mc_psi_mutex_key, rdb_signal_mc_psi_cond_key); #else rdb_bg_thread.init(); rdb_drop_idx_thread.init(); + rdb_mc_thread.init(); #endif mysql_mutex_init(rdb_collation_data_mutex_key, &rdb_collation_data_mutex, MY_MUTEX_INIT_FAST); @@ -4080,6 +4502,8 @@ static int rocksdb_init_func(void *const p) { mysql_mutex_init(rdb_sysvars_psi_mutex_key, &rdb_sysvars_mutex, MY_MUTEX_INIT_FAST); + mysql_mutex_init(rdb_block_cache_resize_mutex_key, + &rdb_block_cache_resize_mutex, MY_MUTEX_INIT_FAST); rdb_open_tables.init_hash(); Rdb_transaction::init_mutex(); @@ -4094,8 +4518,10 @@ static int rocksdb_init_func(void *const p) { rocksdb_hton->rollback = rocksdb_rollback; rocksdb_hton->db_type = DB_TYPE_ROCKSDB; rocksdb_hton->show_status = rocksdb_show_status; + rocksdb_hton->explicit_snapshot = rocksdb_explicit_snapshot; rocksdb_hton->start_consistent_snapshot = rocksdb_start_tx_and_assign_read_view; + rocksdb_hton->start_shared_snapshot = rocksdb_start_tx_with_shared_read_view; rocksdb_hton->savepoint_set = rocksdb_savepoint; rocksdb_hton->savepoint_rollback = rocksdb_rollback_to_savepoint; rocksdb_hton->savepoint_rollback_can_release_mdl = @@ -4158,6 +4584,35 @@ static int rocksdb_init_func(void *const p) { DBUG_RETURN(HA_EXIT_FAILURE); } + // Check whether the filesystem backing rocksdb_datadir allows O_DIRECT + if (rocksdb_db_options->use_direct_reads) { + rocksdb::EnvOptions soptions; + rocksdb::Status check_status; + rocksdb::Env *const env = rocksdb_db_options->env; + + std::string fname = format_string("%s/DIRECT_CHECK", rocksdb_datadir); + if (env->FileExists(fname).ok()) { + std::unique_ptr file; + soptions.use_direct_reads = true; + check_status = env->NewSequentialFile(fname, &file, soptions); + } else { + std::unique_ptr file; + soptions.use_direct_writes = true; + check_status = env->ReopenWritableFile(fname, &file, soptions); + if (file != nullptr) { + file->Close(); + } + env->DeleteFile(fname); + } + + if (!check_status.ok()) { + sql_print_error("RocksDB: Unable to use direct io in rocksdb-datadir:" + "(%s)", check_status.getState()); + rdb_open_tables.free_hash(); + DBUG_RETURN(HA_EXIT_FAILURE); + } + } + if (rocksdb_db_options->allow_mmap_writes && rocksdb_db_options->use_direct_io_for_flush_and_compaction) { // See above comment for allow_mmap_reads. (NO_LINT_DEBUG) @@ -4309,8 +4764,10 @@ static int rocksdb_init_func(void *const p) { cf_options_map->get_defaults()); rocksdb::TransactionDBOptions tx_db_options; - tx_db_options.transaction_lock_timeout = 2; // 2 seconds + tx_db_options.transaction_lock_timeout = 2000; // 2 seconds tx_db_options.custom_mutex_factory = std::make_shared(); + tx_db_options.write_policy = + static_cast(rocksdb_write_policy); status = check_rocksdb_options_compatibility(rocksdb_datadir, main_opts, cf_descr); @@ -4334,7 +4791,7 @@ static int rocksdb_init_func(void *const p) { } cf_manager.init(std::move(cf_options_map), &cf_handles); - if (dict_manager.init(rdb->GetBaseDB(), &cf_manager)) { + if (dict_manager.init(rdb, &cf_manager)) { // NO_LINT_DEBUG sql_print_error("RocksDB: Failed to initialize data dictionary."); rdb_open_tables.free_hash(); @@ -4401,6 +4858,21 @@ static int rocksdb_init_func(void *const p) { DBUG_RETURN(HA_EXIT_FAILURE); } + err = rdb_mc_thread.create_thread(MANUAL_COMPACTION_THREAD_NAME +#ifdef HAVE_PSI_INTERFACE + , + rdb_mc_psi_thread_key +#endif + ); + if (err != 0) { + // NO_LINT_DEBUG + sql_print_error( + "RocksDB: Couldn't start the manual compaction thread: (errno=%d)", + err); + rdb_open_tables.free_hash(); + DBUG_RETURN(HA_EXIT_FAILURE); + } + rdb_set_collation_exception_list(rocksdb_strict_collation_exceptions); if (rocksdb_pause_background_work) { @@ -4496,6 +4968,16 @@ static int rocksdb_done_func(void *const p) { sql_print_error("RocksDB: Couldn't stop the index thread: (errno=%d)", err); } + // signal the manual compaction thread to stop + rdb_mc_thread.signal(true); + // Wait for the manual compaction thread to finish. + err = rdb_mc_thread.join(); + if (err != 0) { + // NO_LINT_DEBUG + sql_print_error( + "RocksDB: Couldn't stop the manual compaction thread: (errno=%d)", err); + } + if (rdb_open_tables.m_hash.records) { // Looks like we are getting unloaded and yet we have some open tables // left behind. @@ -4505,6 +4987,7 @@ static int rocksdb_done_func(void *const p) { rdb_open_tables.free_hash(); mysql_mutex_destroy(&rdb_open_tables.m_mutex); mysql_mutex_destroy(&rdb_sysvars_mutex); + mysql_mutex_destroy(&rdb_block_cache_resize_mutex); delete rdb_collation_exceptions; mysql_mutex_destroy(&rdb_collation_data_mutex); @@ -4709,6 +5192,12 @@ static ulonglong rdb_get_int_col_max_value(const Field *field) { case HA_KEYTYPE_LONGLONG: max_value = 0x7FFFFFFFFFFFFFFFULL; break; + case HA_KEYTYPE_FLOAT: + max_value = 0x1000000ULL; + break; + case HA_KEYTYPE_DOUBLE: + max_value = 0x20000000000000ULL; + break; default: abort(); } @@ -4896,7 +5385,6 @@ longlong ha_rocksdb::update_hidden_pk_val() { /* Get the id of the hidden pk id from m_last_rowkey */ int ha_rocksdb::read_hidden_pk_id_from_rowkey(longlong *const hidden_pk_id) { - DBUG_ASSERT(hidden_pk_id != nullptr); DBUG_ASSERT(table != nullptr); DBUG_ASSERT(has_hidden_pk(table)); @@ -4953,14 +5441,14 @@ ha_rocksdb::ha_rocksdb(my_core::handlerton *const hton, my_core::TABLE_SHARE *const table_arg) : handler(hton, table_arg), m_table_handler(nullptr), m_scan_it(nullptr), m_scan_it_skips_bloom(false), m_scan_it_snapshot(nullptr), + m_scan_it_lower_bound(nullptr), m_scan_it_upper_bound(nullptr), m_tbl_def(nullptr), m_pk_descr(nullptr), m_key_descr_arr(nullptr), m_pk_can_be_decoded(false), m_maybe_unpack_info(false), m_pk_tuple(nullptr), m_pk_packed_tuple(nullptr), m_sk_packed_tuple(nullptr), m_end_key_packed_tuple(nullptr), m_sk_match_prefix(nullptr), m_sk_match_prefix_buf(nullptr), m_sk_packed_tuple_old(nullptr), m_dup_sk_packed_tuple(nullptr), - m_dup_sk_packed_tuple_old(nullptr), m_eq_cond_lower_bound(nullptr), - m_eq_cond_upper_bound(nullptr), m_pack_buffer(nullptr), + m_dup_sk_packed_tuple_old(nullptr), m_pack_buffer(nullptr), m_lock_rows(RDB_LOCK_NONE), m_keyread_only(FALSE), m_encoder_arr(nullptr), m_row_checksums_checked(0), m_in_rpl_delete_rows(false), m_in_rpl_update_rows(false), m_force_skip_unique_check(false) {} @@ -5507,9 +5995,6 @@ int ha_rocksdb::convert_field_from_storage_format( int ha_rocksdb::convert_record_from_storage_format( const rocksdb::Slice *const key, const rocksdb::Slice *const value, uchar *const buf) { - DBUG_ASSERT(key != nullptr); - DBUG_ASSERT(buf != nullptr); - Rdb_string_reader reader(value); /* @@ -5753,7 +6238,6 @@ int ha_rocksdb::alloc_key_buffers(const TABLE *const table_arg, DBUG_ENTER_FUNC(); DBUG_ASSERT(m_pk_tuple == nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); std::shared_ptr *const kd_arr = tbl_def_arg->m_key_descr_arr; @@ -5805,9 +6289,9 @@ int ha_rocksdb::alloc_key_buffers(const TABLE *const table_arg, m_pack_buffer = reinterpret_cast(my_malloc(max_packed_sk_len, MYF(0))); - m_eq_cond_upper_bound = + m_scan_it_lower_bound = reinterpret_cast(my_malloc(max_packed_sk_len, MYF(0))); - m_eq_cond_lower_bound = + m_scan_it_upper_bound = reinterpret_cast(my_malloc(max_packed_sk_len, MYF(0))); /* @@ -5824,7 +6308,7 @@ int ha_rocksdb::alloc_key_buffers(const TABLE *const table_arg, if (m_pk_tuple == nullptr || m_pk_packed_tuple == nullptr || m_sk_packed_tuple == nullptr || m_sk_packed_tuple_old == nullptr || m_end_key_packed_tuple == nullptr || m_pack_buffer == nullptr || - m_eq_cond_upper_bound == nullptr || m_eq_cond_lower_bound == nullptr || + m_scan_it_upper_bound == nullptr || m_scan_it_lower_bound == nullptr || (alloc_alter_buffers && (m_dup_sk_packed_tuple == nullptr || m_dup_sk_packed_tuple_old == nullptr))) { // One or more of the above allocations failed. Clean up and exit @@ -5864,11 +6348,11 @@ void ha_rocksdb::free_key_buffers() { my_free(m_dup_sk_packed_tuple_old); m_dup_sk_packed_tuple_old = nullptr; - my_free(m_eq_cond_upper_bound); - m_eq_cond_upper_bound = nullptr; + my_free(m_scan_it_lower_bound); + m_scan_it_lower_bound = nullptr; - my_free(m_eq_cond_lower_bound); - m_eq_cond_lower_bound = nullptr; + my_free(m_scan_it_upper_bound); + m_scan_it_upper_bound = nullptr; } void ha_rocksdb::set_use_read_free_rpl(const char *const whitelist) { @@ -6081,9 +6565,8 @@ bool ha_rocksdb::get_error_message(const int error, String *const buf) { static_assert(HA_ERR_ROCKSDB_LAST > HA_ERR_LAST, "HA_ERR_ROCKSDB_LAST > HA_ERR_LAST"); - DBUG_ASSERT(buf != nullptr); - - if (error == HA_ERR_LOCK_WAIT_TIMEOUT || error == HA_ERR_LOCK_DEADLOCK) { + if (error == HA_ERR_LOCK_WAIT_TIMEOUT || error == HA_ERR_LOCK_DEADLOCK || + error == HA_ERR_ROCKSDB_STATUS_BUSY) { Rdb_transaction *const tx = get_tx_from_thd(ha_thd()); DBUG_ASSERT(tx != nullptr); buf->append(tx->m_detailed_error); @@ -6165,10 +6648,10 @@ int ha_rocksdb::rdb_error_to_mysql(const rocksdb::Status &s, } if (opt_msg) { - my_error(ER_RDB_STATUS_MSG, MYF(0), opt_msg, s.code(), - s.ToString().c_str()); + std::string concatenated_error = s.ToString() + " (" + std::string(opt_msg) + ")"; + my_error(ER_GET_ERRMSG, MYF(0), s.code(), concatenated_error.c_str(), rocksdb_hton_name); } else { - my_error(ER_RDB_STATUS_GENERAL, MYF(0), s.code(), s.ToString().c_str()); + my_error(ER_GET_ERRMSG, MYF(0), s.code(), s.ToString().c_str(), rocksdb_hton_name); } return err; @@ -6213,7 +6696,6 @@ int ha_rocksdb::create_key_defs( /* = nullptr */) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); uint i; @@ -6284,9 +6766,7 @@ int ha_rocksdb::create_cfs( std::array *const cfs) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); char tablename_sys[NAME_LEN + 1]; @@ -6315,11 +6795,22 @@ int ha_rocksdb::create_cfs( } collation_err += coll->name; } - my_error(ER_UNSUPPORTED_COLLATION, MYF(0), - tbl_def_arg->full_tablename().c_str(), - table_arg->key_info[i].key_part[part].field->field_name, - collation_err.c_str()); - DBUG_RETURN(HA_EXIT_FAILURE); + + if (rocksdb_error_on_suboptimal_collation) { + my_error(ER_UNSUPPORTED_COLLATION, MYF(0), + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name, + collation_err.c_str()); + DBUG_RETURN(HA_EXIT_FAILURE); + } else { + push_warning_printf( + ha_thd(), Sql_condition::WARN_LEVEL_WARN, ER_WRONG_ARGUMENTS, + "Unsupported collation on string indexed column %s.%s Use " + "binary collation (%s).", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name, + collation_err.c_str()); + } } } } @@ -6380,10 +6871,6 @@ int ha_rocksdb::create_inplace_key_defs( const std::array &cfs) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(table_arg != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); - DBUG_ASSERT(old_tbl_def_arg != nullptr); - std::shared_ptr *const old_key_descr = old_tbl_def_arg->m_key_descr_arr; std::shared_ptr *const new_key_descr = @@ -6451,11 +6938,6 @@ std::unordered_map ha_rocksdb::get_old_key_positions( const Rdb_tbl_def *const old_tbl_def_arg) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(table_arg != nullptr); - DBUG_ASSERT(old_table_arg != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); - DBUG_ASSERT(old_tbl_def_arg != nullptr); - std::shared_ptr *const old_key_descr = old_tbl_def_arg->m_key_descr_arr; std::unordered_map old_key_pos; @@ -6521,9 +7003,6 @@ int ha_rocksdb::compare_keys(const KEY *const old_key, const KEY *const new_key) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(old_key != nullptr); - DBUG_ASSERT(new_key != nullptr); - /* Check index name. */ if (strcmp(old_key->name, new_key->name) != 0) { DBUG_RETURN(HA_EXIT_FAILURE); @@ -6554,9 +7033,6 @@ int ha_rocksdb::compare_key_parts(const KEY *const old_key, const KEY *const new_key) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(old_key != nullptr); - DBUG_ASSERT(new_key != nullptr); - /* Skip if key parts do not match, as it is a different key */ if (new_key->user_defined_key_parts != old_key->user_defined_key_parts) { DBUG_RETURN(HA_EXIT_FAILURE); @@ -6601,7 +7077,6 @@ int ha_rocksdb::create_key_def(const TABLE *const table_arg, const uint &i, const struct key_def_cf_info &cf_info) const { DBUG_ENTER_FUNC(); - DBUG_ASSERT(new_key_def != nullptr); DBUG_ASSERT(*new_key_def == nullptr); uint64 ttl_duration = 0; @@ -6688,8 +7163,6 @@ int ha_rocksdb::create_key_def(const TABLE *const table_arg, const uint &i, int rdb_normalize_tablename(const std::string &tablename, std::string *const strbuf) { - DBUG_ASSERT(strbuf != nullptr); - if (tablename.size() < 2 || tablename[0] != '.' || tablename[1] != '/') { DBUG_ASSERT(0); // We were not passed table name? return HA_ERR_ROCKSDB_INVALID_TABLE; @@ -7004,8 +7477,6 @@ int ha_rocksdb::read_key_exact(const Rdb_key_def &kd, const bool &full_key_match, const rocksdb::Slice &key_slice, const int64_t ttl_filter_ts) { - DBUG_ASSERT(iter != nullptr); - /* We are looking for the first record such that index_tuple= lookup_tuple. @@ -7213,8 +7684,6 @@ int ha_rocksdb::calc_eq_cond_len(const Rdb_key_def &kd, } int ha_rocksdb::read_row_from_primary_key(uchar *const buf) { - DBUG_ASSERT(buf != nullptr); - int rc; const rocksdb::Slice &rkey = m_scan_it->key(); const uint pk_size = rkey.size(); @@ -7238,8 +7707,6 @@ int ha_rocksdb::read_row_from_primary_key(uchar *const buf) { int ha_rocksdb::read_row_from_secondary_key(uchar *const buf, const Rdb_key_def &kd, bool move_forward) { - DBUG_ASSERT(buf != nullptr); - int rc = 0; uint pk_size; @@ -7342,7 +7809,6 @@ ulong ha_rocksdb::index_flags(uint inx, uint part, bool all_parts) const { pair for. */ int ha_rocksdb::secondary_index_read(const int keyno, uchar *const buf) { - DBUG_ASSERT(buf != nullptr); DBUG_ASSERT(table != nullptr); stats.rows_requested++; @@ -7612,7 +8078,7 @@ int ha_rocksdb::index_read_map_impl(uchar *const buf, const uchar *const key, else rc = read_row_from_secondary_key(buf, kd, move_forward); - if (rc != HA_ERR_LOCK_DEADLOCK || !is_new_snapshot) + if (rc != HA_ERR_ROCKSDB_STATUS_BUSY || !is_new_snapshot) break; /* Exit the loop */ // release the snapshot and iterator so they will be regenerated @@ -7655,8 +8121,6 @@ int ha_rocksdb::index_read_map_impl(uchar *const buf, const uchar *const key, int ha_rocksdb::find_icp_matching_index_rec(const bool &move_forward, uchar *const buf) { - DBUG_ASSERT(buf != nullptr); - if (pushed_idx_cond && pushed_idx_cond_keyno == active_index) { const Rdb_key_def &kd = *m_key_descr_arr[active_index]; @@ -7974,8 +8438,6 @@ int ha_rocksdb::get_row_by_rowid(uchar *const buf, const char *const rowid, const bool skip_ttl_check) { DBUG_ENTER_FUNC(); - DBUG_ASSERT(buf != nullptr); - DBUG_ASSERT(rowid != nullptr); DBUG_ASSERT(table != nullptr); int rc; @@ -8190,8 +8652,6 @@ int ha_rocksdb::index_last(uchar *const buf) { int ha_rocksdb::index_first_intern(uchar *const buf) { DBUG_ENTER_FUNC(); - DBUG_ASSERT(buf != nullptr); - uchar *key; uint key_size; int rc; @@ -8205,7 +8665,7 @@ int ha_rocksdb::index_first_intern(uchar *const buf) { DBUG_ASSERT(key != nullptr); const Rdb_key_def &kd = *m_key_descr_arr[active_index]; - kd.get_first_key(key, &key_size); + int key_start_matching_bytes = kd.get_first_key(key, &key_size); rocksdb::Slice index_key((const char *)key, key_size); @@ -8216,12 +8676,12 @@ int ha_rocksdb::index_first_intern(uchar *const buf) { // Loop as long as we get a deadlock error AND we end up creating the // snapshot here (i.e. it did not exist prior to this) for (;;) { - setup_scan_iterator(kd, &index_key, false, Rdb_key_def::INDEX_NUMBER_SIZE); + setup_scan_iterator(kd, &index_key, false, key_start_matching_bytes); m_scan_it->Seek(index_key); m_skip_scan_it_next_call = true; rc = index_next_with_direction(buf, true); - if (rc != HA_ERR_LOCK_DEADLOCK || !is_new_snapshot) + if (rc != HA_ERR_ROCKSDB_STATUS_BUSY || !is_new_snapshot) break; // exit the loop // release the snapshot and iterator so they will be regenerated @@ -8281,8 +8741,6 @@ int ha_rocksdb::index_first_intern(uchar *const buf) { int ha_rocksdb::index_last_intern(uchar *const buf) { DBUG_ENTER_FUNC(); - DBUG_ASSERT(buf != nullptr); - uchar *key; uint key_size; int rc; @@ -8324,7 +8782,7 @@ int ha_rocksdb::index_last_intern(uchar *const buf) { rc = secondary_index_read(active_index, buf); } - if (rc != HA_ERR_LOCK_DEADLOCK || !is_new_snapshot) + if (rc != HA_ERR_ROCKSDB_STATUS_BUSY || !is_new_snapshot) break; /* exit the loop */ // release the snapshot and iterator so they will be regenerated @@ -8405,7 +8863,6 @@ bool ha_rocksdb::commit_in_the_middle() { @retval false if bulk commit was skipped or succeeded */ bool ha_rocksdb::do_bulk_commit(Rdb_transaction *const tx) { - DBUG_ASSERT(tx != nullptr); return commit_in_the_middle() && tx->get_write_count() >= THDVAR(table->in_use, bulk_load_size) && tx->flush_batch(); @@ -8418,7 +8875,6 @@ bool ha_rocksdb::do_bulk_commit(Rdb_transaction *const tx) { 'auto-incremented' pk.) */ bool ha_rocksdb::has_hidden_pk(const TABLE *const table) const { - DBUG_ASSERT(table != nullptr); return Rdb_key_def::table_has_hidden_pk(table); } @@ -8428,9 +8884,7 @@ bool ha_rocksdb::has_hidden_pk(const TABLE *const table) const { */ bool ha_rocksdb::is_hidden_pk(const uint index, const TABLE *const table_arg, const Rdb_tbl_def *const tbl_def_arg) { - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); return (table_arg->s->primary_key == MAX_INDEXES && index == tbl_def_arg->m_key_count - 1); @@ -8439,9 +8893,7 @@ bool ha_rocksdb::is_hidden_pk(const uint index, const TABLE *const table_arg, /* Returns index of primary key */ uint ha_rocksdb::pk_index(const TABLE *const table_arg, const Rdb_tbl_def *const tbl_def_arg) { - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); return table_arg->s->primary_key == MAX_INDEXES ? tbl_def_arg->m_key_count - 1 : table_arg->s->primary_key; @@ -8450,9 +8902,7 @@ uint ha_rocksdb::pk_index(const TABLE *const table_arg, /* Returns true if given index number is a primary key */ bool ha_rocksdb::is_pk(const uint index, const TABLE *const table_arg, const Rdb_tbl_def *const tbl_def_arg) { - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); return index == table_arg->s->primary_key || is_hidden_pk(index, table_arg, tbl_def_arg); @@ -8467,9 +8917,6 @@ uint ha_rocksdb::max_supported_key_part_length() const { const char *ha_rocksdb::get_key_name(const uint index, const TABLE *const table_arg, const Rdb_tbl_def *const tbl_def_arg) { - DBUG_ASSERT(table_arg != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); - if (is_hidden_pk(index, table_arg, tbl_def_arg)) { return HIDDEN_PK_NAME; } @@ -8483,9 +8930,6 @@ const char *ha_rocksdb::get_key_name(const uint index, const char *ha_rocksdb::get_key_comment(const uint index, const TABLE *const table_arg, const Rdb_tbl_def *const tbl_def_arg) { - DBUG_ASSERT(table_arg != nullptr); - DBUG_ASSERT(tbl_def_arg != nullptr); - if (is_hidden_pk(index, table_arg, tbl_def_arg)) { return nullptr; } @@ -8537,7 +8981,6 @@ const std::string ha_rocksdb::generate_cf_name(const uint index, } const std::string ha_rocksdb::get_table_comment(const TABLE *const table_arg) { - DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); return table_arg->s->comment.str; @@ -8651,8 +9094,7 @@ int ha_rocksdb::check_and_lock_unique_pk(const uint &key_id, /* If the keys are the same, then no lock is needed */ - if (!Rdb_pk_comparator::bytewise_compare(row_info.new_pk_slice, - row_info.old_pk_slice)) { + if (!row_info.new_pk_slice.compare(row_info.old_pk_slice)) { *found = false; return HA_EXIT_SUCCESS; } @@ -8741,8 +9183,7 @@ int ha_rocksdb::check_and_lock_sk(const uint &key_id, rocksdb::Slice((const char *)m_sk_packed_tuple, size); /* - For UPDATEs, if the key has changed, we need to obtain a lock. INSERTs - always require locking. + Acquire lock on the old key in case of UPDATE */ if (row_info.old_data != nullptr) { size = kd.pack_record(table, m_pack_buffer, row_info.old_data, @@ -8751,14 +9192,18 @@ int ha_rocksdb::check_and_lock_sk(const uint &key_id, const rocksdb::Slice old_slice = rocksdb::Slice((const char *)m_sk_packed_tuple_old, size); - /* - For updates, if the keys are the same, then no lock is needed + const rocksdb::Status s = + get_for_update(row_info.tx, kd.get_cf(), old_slice, nullptr); + if (!s.ok()) { + return row_info.tx->set_status_error(table->in_use, s, kd, m_tbl_def, + m_table_handler); + } - Also check to see if the key has any fields set to NULL. If it does, then - this key is unique since NULL is not equal to each other, so no lock is - needed. + /* + If the old and new keys are the same we're done since we've already taken + the lock on the old key */ - if (!Rdb_pk_comparator::bytewise_compare(new_slice, old_slice)) { + if (!new_slice.compare(old_slice)) { return HA_EXIT_SUCCESS; } } @@ -8784,8 +9229,14 @@ int ha_rocksdb::check_and_lock_sk(const uint &key_id, The bloom filter may need to be disabled for this lookup. */ + uchar lower_bound_buf[Rdb_key_def::INDEX_NUMBER_SIZE]; + uchar upper_bound_buf[Rdb_key_def::INDEX_NUMBER_SIZE]; + rocksdb::Slice lower_bound_slice; + rocksdb::Slice upper_bound_slice; + const bool total_order_seek = !check_bloom_and_set_bounds( - ha_thd(), kd, new_slice, all_parts_used); + ha_thd(), kd, new_slice, all_parts_used, Rdb_key_def::INDEX_NUMBER_SIZE, + lower_bound_buf, upper_bound_buf, &lower_bound_slice, &upper_bound_slice); const bool fill_cache = !THDVAR(ha_thd(), skip_fill_cache); const rocksdb::Status s = @@ -8796,9 +9247,8 @@ int ha_rocksdb::check_and_lock_sk(const uint &key_id, } rocksdb::Iterator *const iter = row_info.tx->get_iterator( - kd.get_cf(), total_order_seek, fill_cache, - m_eq_cond_lower_bound_slice, m_eq_cond_upper_bound_slice, - true /* read current data */, + kd.get_cf(), total_order_seek, fill_cache, lower_bound_slice, + upper_bound_slice, true /* read current data */, false /* acquire snapshot */); /* Need to scan the transaction to see if there is a duplicate key. @@ -8960,7 +9410,7 @@ int ha_rocksdb::update_pk(const Rdb_key_def &kd, } } - if (table->next_number_field) { + if (table->found_next_number_field) { update_auto_incr_val_from_field(); } @@ -9008,9 +9458,11 @@ int ha_rocksdb::update_pk(const Rdb_key_def &kd, } int ha_rocksdb::update_sk(const TABLE *const table_arg, const Rdb_key_def &kd, - const struct update_row_info &row_info) { + const struct update_row_info &row_info, + const bool bulk_load_sk) { int new_packed_size; int old_packed_size; + int rc = HA_EXIT_SUCCESS; rocksdb::Slice new_key_slice; rocksdb::Slice new_value_slice; @@ -9088,18 +9540,23 @@ int ha_rocksdb::update_sk(const TABLE *const table_arg, const Rdb_key_def &kd, rocksdb::Slice(reinterpret_cast(m_sk_tails.ptr()), m_sk_tails.get_current_pos()); - row_info.tx->get_indexed_write_batch()->Put(kd.get_cf(), new_key_slice, - new_value_slice); + if (bulk_load_sk && row_info.old_data == nullptr) { + rc = bulk_load_key(row_info.tx, kd, new_key_slice, new_value_slice, true); + } else { + row_info.tx->get_indexed_write_batch()->Put(kd.get_cf(), new_key_slice, + new_value_slice); + } row_info.tx->update_bytes_written(bytes_written + new_key_slice.size() + new_value_slice.size()); - return HA_EXIT_SUCCESS; + return rc; } int ha_rocksdb::update_indexes(const struct update_row_info &row_info, const bool &pk_changed) { int rc; + bool bulk_load_sk; // The PK must be updated first to pull out the TTL value. rc = update_pk(*m_pk_descr, row_info, pk_changed); @@ -9107,13 +9564,17 @@ int ha_rocksdb::update_indexes(const struct update_row_info &row_info, return rc; } - // Update the remaining indexes. + // Update the remaining indexes. Allow bulk loading only if + // allow_sk is enabled + bulk_load_sk = rocksdb_enable_bulk_load_api && + THDVAR(table->in_use, bulk_load) && + THDVAR(table->in_use, bulk_load_allow_sk); for (uint key_id = 0; key_id < m_tbl_def->m_key_count; key_id++) { if (is_pk(key_id, table, m_tbl_def)) { continue; } - rc = update_sk(table, *m_key_descr_arr[key_id], row_info); + rc = update_sk(table, *m_key_descr_arr[key_id], row_info, bulk_load_sk); if (rc != HA_EXIT_SUCCESS) { return rc; } @@ -9206,25 +9667,22 @@ int ha_rocksdb::update_write_row(const uchar *const old_data, 0x0000b3eb003f65c5e78857, and lower bound would be 0x0000b3eb003f65c5e78859. These cover given eq condition range. */ -void ha_rocksdb::setup_iterator_bounds(const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond) { - uint eq_cond_len = eq_cond.size(); - DBUG_ASSERT(eq_cond_len >= Rdb_key_def::INDEX_NUMBER_SIZE); - memcpy(m_eq_cond_upper_bound, eq_cond.data(), eq_cond_len); - kd.successor(m_eq_cond_upper_bound, eq_cond_len); - memcpy(m_eq_cond_lower_bound, eq_cond.data(), eq_cond_len); - kd.predecessor(m_eq_cond_lower_bound, eq_cond_len); +void ha_rocksdb::setup_iterator_bounds( + const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, size_t bound_len, + uchar *const lower_bound, uchar *const upper_bound, + rocksdb::Slice *lower_bound_slice, rocksdb::Slice *upper_bound_slice) { + uint min_len = std::min(eq_cond.size(), bound_len); + memcpy(upper_bound, eq_cond.data(), min_len); + kd.successor(upper_bound, min_len); + memcpy(lower_bound, eq_cond.data(), min_len); + kd.predecessor(lower_bound, min_len); if (kd.m_is_reverse_cf) { - m_eq_cond_upper_bound_slice = - rocksdb::Slice((const char *)m_eq_cond_lower_bound, eq_cond_len); - m_eq_cond_lower_bound_slice = - rocksdb::Slice((const char *)m_eq_cond_upper_bound, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); } else { - m_eq_cond_upper_bound_slice = - rocksdb::Slice((const char *)m_eq_cond_upper_bound, eq_cond_len); - m_eq_cond_lower_bound_slice = - rocksdb::Slice((const char *)m_eq_cond_lower_bound, eq_cond_len); + *upper_bound_slice = rocksdb::Slice((const char *)upper_bound, min_len); + *lower_bound_slice = rocksdb::Slice((const char *)lower_bound, min_len); } } @@ -9236,7 +9694,6 @@ void ha_rocksdb::setup_scan_iterator(const Rdb_key_def &kd, rocksdb::Slice *const slice, const bool use_all_keys, const uint eq_cond_len) { - DBUG_ASSERT(slice != nullptr); DBUG_ASSERT(slice->size() >= eq_cond_len); Rdb_transaction *const tx = get_or_create_tx(table->in_use); @@ -9244,7 +9701,10 @@ void ha_rocksdb::setup_scan_iterator(const Rdb_key_def &kd, bool skip_bloom = true; const rocksdb::Slice eq_cond(slice->data(), eq_cond_len); - if (check_bloom_and_set_bounds(ha_thd(), kd, eq_cond, use_all_keys)) { + if (check_bloom_and_set_bounds( + ha_thd(), kd, eq_cond, use_all_keys, eq_cond_len, + m_scan_it_lower_bound, m_scan_it_upper_bound, + &m_scan_it_lower_bound_slice, &m_scan_it_upper_bound_slice)) { skip_bloom = false; } @@ -9285,8 +9745,8 @@ void ha_rocksdb::setup_scan_iterator(const Rdb_key_def &kd, m_scan_it = rdb->NewIterator(read_opts, kd.get_cf()); } else { m_scan_it = tx->get_iterator(kd.get_cf(), skip_bloom, fill_cache, - m_eq_cond_lower_bound_slice, - m_eq_cond_upper_bound_slice); + m_scan_it_lower_bound_slice, + m_scan_it_upper_bound_slice); } m_scan_it_skips_bloom = skip_bloom; } @@ -9305,12 +9765,12 @@ void ha_rocksdb::release_scan_iterator() { void ha_rocksdb::setup_iterator_for_rnd_scan() { uint key_size; - m_pk_descr->get_first_key(m_pk_packed_tuple, &key_size); + int key_start_matching_bytes = m_pk_descr->get_first_key(m_pk_packed_tuple, &key_size); rocksdb::Slice table_key((const char *)m_pk_packed_tuple, key_size); setup_scan_iterator(*m_pk_descr, &table_key, false, - Rdb_key_def::INDEX_NUMBER_SIZE); + key_start_matching_bytes); m_scan_it->Seek(table_key); m_skip_scan_it_next_call = true; } @@ -9353,7 +9813,7 @@ int ha_rocksdb::rnd_next(uchar *const buf) { ha_statistic_increment(&SSV::ha_read_rnd_next_count); for (;;) { rc = rnd_next_with_direction(buf, true); - if (rc != HA_ERR_LOCK_DEADLOCK || !m_rnd_scan_is_new_snapshot) + if (rc != HA_ERR_ROCKSDB_STATUS_BUSY || !m_rnd_scan_is_new_snapshot) break; /* exit the loop */ // release the snapshot and iterator and then regenerate them @@ -9676,7 +10136,7 @@ int ha_rocksdb::info(uint flag) { if (static_cast(stats.data_file_length) < 0 || static_cast(stats.index_file_length) < 0 || static_cast(stats.records) < 0) { - if (analyze(nullptr, nullptr)) { + if (calculate_stats_for_table()) { DBUG_RETURN(HA_EXIT_FAILURE); } @@ -9756,10 +10216,21 @@ int ha_rocksdb::info(uint flag) { KEY *const k = &table->key_info[i]; for (uint j = 0; j < k->actual_key_parts; j++) { const Rdb_index_stats &k_stats = m_key_descr_arr[i]->m_stats; - uint x = k_stats.m_distinct_keys_per_prefix.size() > j && - k_stats.m_distinct_keys_per_prefix[j] > 0 - ? k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j] - : 0; + uint x; + + if (k_stats.m_distinct_keys_per_prefix.size() > j && + k_stats.m_distinct_keys_per_prefix[j] > 0) { + x = k_stats.m_rows / k_stats.m_distinct_keys_per_prefix[j]; + /* + If the number of rows is less than the number of prefixes (due to + sampling), the average number of rows with the same prefix is 1. + */ + if (x == 0) { + x = 1; + } + } else { + x = 0; + } if (x > stats.records) x = stats.records; if ((x == 0 && rocksdb_debug_optimizer_no_zero_cardinality) || @@ -9932,7 +10403,7 @@ THR_LOCK_DATA **ha_rocksdb::store_lock(THD *const thd, THR_LOCK_DATA **to, m_lock_rows = RDB_LOCK_WRITE; } else if (lock_type == TL_READ_WITH_SHARED_LOCKS) { m_lock_rows = RDB_LOCK_READ; - } else { + } else if (lock_type != TL_IGNORE) { m_lock_rows = RDB_LOCK_NONE; if (THDVAR(thd, lock_scanned_rows)) { /* @@ -10023,22 +10494,24 @@ int ha_rocksdb::external_lock(THD *const thd, int lock_type) { } if (lock_type == F_UNLCK) { - Rdb_transaction *const tx = get_or_create_tx(thd); + Rdb_transaction *const tx = get_tx_from_thd(thd); - tx->io_perf_end_and_record(&m_io_perf); - tx->m_n_mysql_tables_in_use--; - if (tx->m_n_mysql_tables_in_use == 0 && - !my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) { - /* - Do like InnoDB: when we get here, it's time to commit a - single-statement transaction. + if (tx) { + tx->io_perf_end_and_record(&m_io_perf); + tx->m_n_mysql_tables_in_use--; + if (tx->m_n_mysql_tables_in_use == 0 && + !my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) { + /* + Do like InnoDB: when we get here, it's time to commit a + single-statement transaction. - If the statement involved multiple tables, this code will be executed - for each of them, but that's ok because non-first tx->commit() calls - will be no-ops. - */ - if (tx->commit_or_rollback()) { - res = HA_ERR_INTERNAL_ERROR; + If the statement involved multiple tables, this code will be executed + for each of them, but that's ok because non-first tx->commit() calls + will be no-ops. + */ + if (tx->commit_or_rollback()) { + res = HA_ERR_INTERNAL_ERROR; + } } } } else { @@ -10075,6 +10548,11 @@ int ha_rocksdb::external_lock(THD *const thd, int lock_type) { DBUG_RETURN(HA_ERR_UNSUPPORTED); } + if (thd->get_explicit_snapshot()) { + my_error(ER_UPDATES_WITH_EXPLICIT_SNAPSHOT, MYF(0)); + DBUG_RETURN(HA_ERR_UNSUPPORTED); + } + /* SQL layer signals us to take a write lock. It does so when starting DML statement. We should put locks on the rows we're reading. @@ -10282,8 +10760,6 @@ void Rdb_drop_index_thread::run() { } Rdb_tbl_def *ha_rocksdb::get_table_if_exists(const char *const tablename) { - DBUG_ASSERT(tablename != nullptr); - std::string str; if (rdb_normalize_tablename(tablename, &str) != HA_EXIT_SUCCESS) { // We were not passed table name? @@ -10346,6 +10822,12 @@ int ha_rocksdb::remove_rows(Rdb_tbl_def *const tbl) { char key_buf[MAX_KEY_LENGTH]; uint key_len; ulonglong bytes_written = 0; + + uchar lower_bound_buf[Rdb_key_def::INDEX_NUMBER_SIZE]; + uchar upper_bound_buf[Rdb_key_def::INDEX_NUMBER_SIZE]; + rocksdb::Slice lower_bound_slice; + rocksdb::Slice upper_bound_slice; + /* Remove all records in each index. (This is is not crash-safe, but it doesn't matter, because bulk row @@ -10356,9 +10838,12 @@ int ha_rocksdb::remove_rows(Rdb_tbl_def *const tbl) { kd.get_infimum_key(reinterpret_cast(key_buf), &key_len); rocksdb::ColumnFamilyHandle *cf = kd.get_cf(); const rocksdb::Slice table_key(key_buf, key_len); - setup_iterator_bounds(kd, table_key); - opts.iterate_lower_bound = &m_eq_cond_lower_bound_slice; - opts.iterate_upper_bound = &m_eq_cond_upper_bound_slice; + setup_iterator_bounds(kd, table_key, Rdb_key_def::INDEX_NUMBER_SIZE, + lower_bound_buf, upper_bound_buf, &lower_bound_slice, + &upper_bound_slice); + DBUG_ASSERT(key_len == Rdb_key_def::INDEX_NUMBER_SIZE); + opts.iterate_lower_bound = &lower_bound_slice; + opts.iterate_upper_bound = &upper_bound_slice; std::unique_ptr it(rdb->NewIterator(opts, cf)); it->Seek(table_key); @@ -10397,9 +10882,6 @@ int ha_rocksdb::remove_rows(Rdb_tbl_def *const tbl) { int ha_rocksdb::rename_table(const char *const from, const char *const to) { DBUG_ENTER_FUNC(); - DBUG_ASSERT(from != nullptr); - DBUG_ASSERT(to != nullptr); - std::string from_str; std::string to_str; std::string from_db; @@ -10652,29 +11134,28 @@ int ha_rocksdb::optimize(THD *const thd, HA_CHECK_OPT *const check_opt) { DBUG_RETURN(HA_EXIT_SUCCESS); } -int ha_rocksdb::calculate_stats(const TABLE *const table_arg, THD *const thd, - HA_CHECK_OPT *const check_opt) { +static int calculate_stats( + const std::unordered_map> + &to_recalc, + bool include_memtables) { DBUG_ENTER_FUNC(); // find per column family key ranges which need to be queried std::unordered_map> ranges; - std::unordered_set ids_to_check; - std::vector buf(table_arg->s->keys * 2 * - Rdb_key_def::INDEX_NUMBER_SIZE); std::unordered_map stats; - for (uint i = 0; i < table_arg->s->keys; i++) { - const auto bufp = &buf[i * 2 * Rdb_key_def::INDEX_NUMBER_SIZE]; - const Rdb_key_def &kd = *m_key_descr_arr[i]; - const GL_INDEX_ID index_id = kd.get_gl_index_id(); - ranges[kd.get_cf()].push_back(get_range(i, bufp)); + std::vector buf(to_recalc.size() * 2 * Rdb_key_def::INDEX_NUMBER_SIZE); + + uchar *bufp = buf.data(); + for (const auto &it : to_recalc) { + const GL_INDEX_ID index_id = it.first; + auto &kd = it.second; + ranges[kd->get_cf()].push_back(myrocks::get_range(*kd, bufp)); + bufp += 2 * Rdb_key_def::INDEX_NUMBER_SIZE; - ids_to_check.insert(index_id); - // Initialize the stats to 0. If there are no files that contain - // this gl_index_id, then 0 should be stored for the cached stats. stats[index_id] = Rdb_index_stats(index_id); - DBUG_ASSERT(kd.get_key_parts() > 0); - stats[index_id].m_distinct_keys_per_prefix.resize(kd.get_key_parts()); + DBUG_ASSERT(kd->get_key_parts() > 0); + stats[index_id].m_distinct_keys_per_prefix.resize(kd->get_key_parts()); } // get RocksDB table properties for these ranges @@ -10685,8 +11166,8 @@ int ha_rocksdb::calculate_stats(const TABLE *const table_arg, THD *const thd, it.first, &it.second[0], it.second.size(), &props); DBUG_ASSERT(props.size() >= old_size); if (!status.ok()) { - DBUG_RETURN( - rdb_error_to_mysql(status, "Could not access RocksDB properties")); + DBUG_RETURN(ha_rocksdb::rdb_error_to_mysql( + status, "Could not access RocksDB properties")); } } @@ -10707,61 +11188,62 @@ int ha_rocksdb::calculate_stats(const TABLE *const table_arg, THD *const thd, other SQL tables, it can be that we're only seeing a small fraction of table's entries (and so we can't update statistics based on that). */ - if (ids_to_check.find(it1.m_gl_index_id) == ids_to_check.end()) + if (stats.find(it1.m_gl_index_id) == stats.end()) { continue; + } - auto kd = ddl_manager.safe_find(it1.m_gl_index_id); - DBUG_ASSERT(kd != nullptr); - stats[it1.m_gl_index_id].merge(it1, true, kd->max_storage_fmt_length()); + auto it_index = to_recalc.find(it1.m_gl_index_id); + DBUG_ASSERT(it_index != to_recalc.end()); + if (it_index == to_recalc.end()) { + continue; + } + stats[it1.m_gl_index_id].merge( + it1, true, it_index->second->max_storage_fmt_length()); } num_sst++; } - // calculate memtable cardinality - Rdb_tbl_card_coll cardinality_collector(rocksdb_table_stats_sampling_pct); - auto read_opts = rocksdb::ReadOptions(); - read_opts.read_tier = rocksdb::ReadTier::kMemtableTier; - for (uint i = 0; i < table_arg->s->keys; i++) { - const Rdb_key_def &kd = *m_key_descr_arr[i]; - Rdb_index_stats &stat = stats[kd.get_gl_index_id()]; + if (include_memtables) { + // calculate memtable cardinality + Rdb_tbl_card_coll cardinality_collector(rocksdb_table_stats_sampling_pct); + auto read_opts = rocksdb::ReadOptions(); + read_opts.read_tier = rocksdb::ReadTier::kMemtableTier; + for (const auto &it_kd : to_recalc) { + const std::shared_ptr &kd = it_kd.second; + Rdb_index_stats &stat = stats[kd->get_gl_index_id()]; - uchar r_buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2]; - auto r = get_range(i, r_buf); - uint64_t memtableCount; - uint64_t memtableSize; - rdb->GetApproximateMemTableStats(kd.get_cf(), r, &memtableCount, - &memtableSize); - if (memtableCount < (uint64_t)stat.m_rows / 10) { - // skip tables that already have enough stats from SST files to reduce - // overhead and avoid degradation of big tables stats by sampling from - // relatively tiny (less than 10% of full data set) memtable dataset - continue; - } - - std::unique_ptr it = std::unique_ptr( - rdb->NewIterator(read_opts, kd.get_cf())); - - uchar *first_key; - uint key_size; - if (is_pk(i, table, m_tbl_def)) { - first_key = m_pk_packed_tuple; - } else { - first_key = m_sk_packed_tuple; - } - kd.get_first_key(first_key, &key_size); - rocksdb::Slice first_index_key((const char *)first_key, key_size); - - cardinality_collector.Reset(); - for (it->Seek(first_index_key); is_valid(it.get()); it->Next()) { - const rocksdb::Slice key = it->key(); - if (!kd.covers_key(key)) { - break; // end of this index + uchar r_buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2]; + auto r = myrocks::get_range(*kd, r_buf); + uint64_t memtableCount; + uint64_t memtableSize; + rdb->GetApproximateMemTableStats(kd->get_cf(), r, &memtableCount, + &memtableSize); + if (memtableCount < (uint64_t)stat.m_rows / 10) { + // skip tables that already have enough stats from SST files to reduce + // overhead and avoid degradation of big tables stats by sampling from + // relatively tiny (less than 10% of full data set) memtable dataset + continue; } - stat.m_rows++; - cardinality_collector.ProcessKey(key, &kd, &stat); + std::unique_ptr it = + std::unique_ptr( + rdb->NewIterator(read_opts, kd->get_cf())); + + rocksdb::Slice first_index_key((const char *)r_buf, + Rdb_key_def::INDEX_NUMBER_SIZE); + + cardinality_collector.Reset(); + for (it->Seek(first_index_key); is_valid(it.get()); it->Next()) { + const rocksdb::Slice key = it->key(); + if (!kd->covers_key(key)) { + break; // end of this index + } + stat.m_rows++; + + cardinality_collector.ProcessKey(key, kd.get(), &stat); + } + cardinality_collector.AdjustStats(&stat); } - cardinality_collector.AdjustStats(&stat); } // set and persist new stats @@ -10771,6 +11253,19 @@ int ha_rocksdb::calculate_stats(const TABLE *const table_arg, THD *const thd, DBUG_RETURN(HA_EXIT_SUCCESS); } +int ha_rocksdb::calculate_stats_for_table() { + DBUG_ENTER_FUNC(); + + std::unordered_map> + ids_to_check; + for (uint i = 0; i < table->s->keys; i++) { + ids_to_check.insert(std::make_pair(m_key_descr_arr[i]->get_gl_index_id(), + m_key_descr_arr[i])); + } + + DBUG_RETURN(calculate_stats(ids_to_check, true)); +} + /* @return HA_ADMIN_OK OK @@ -10779,7 +11274,16 @@ int ha_rocksdb::calculate_stats(const TABLE *const table_arg, THD *const thd, int ha_rocksdb::analyze(THD *const thd, HA_CHECK_OPT *const check_opt) { DBUG_ENTER_FUNC(); - if (table && calculate_stats(table, thd, check_opt) != HA_EXIT_SUCCESS) { + if (table) { + if (calculate_stats_for_table() != HA_EXIT_SUCCESS) { + DBUG_RETURN(HA_ADMIN_FAILED); + } + } + + // A call to ::info is needed to repopulate some SQL level structs. This is + // necessary for online analyze because we cannot rely on another ::open + // call to call info for us. + if (info(HA_STATUS_CONST | HA_STATUS_VARIABLE) != HA_EXIT_SUCCESS) { DBUG_RETURN(HA_ADMIN_FAILED); } @@ -11184,7 +11688,9 @@ bool ha_rocksdb::prepare_inplace_alter_table( if (!new_tdef) { new_tdef = m_tbl_def; } - max_auto_incr = load_auto_incr_value_from_index(); + if (table->found_next_number_field) { + max_auto_incr = load_auto_incr_value_from_index(); + } } ha_alter_info->handler_ctx = new Rdb_inplace_alter_ctx( @@ -11636,18 +12142,6 @@ bool ha_rocksdb::commit_inplace_alter_table( dict_manager.finish_indexes_operation( create_index_ids, Rdb_key_def::DDL_CREATE_INDEX_ONGOING); - /* - We need to recalculate the index stats here manually. The reason is that - the secondary index does not exist inside - m_index_num_to_keydef until it is committed to the data dictionary, which - prevents us from updating the stats normally as the ddl_manager cannot - find the proper gl_index_ids yet during adjust_stats calls. - */ - if (calculate_stats(altered_table, nullptr, nullptr)) { - /* Failed to update index statistics, should never happen */ - DBUG_ASSERT(0); - } - rdb_drop_idx_thread.signal(); } @@ -11719,6 +12213,8 @@ struct rocksdb_status_counters_t { uint64_t block_cache_data_hit; uint64_t block_cache_data_add; uint64_t bloom_filter_useful; + uint64_t bloom_filter_full_positive; + uint64_t bloom_filter_full_true_positive; uint64_t memtable_hit; uint64_t memtable_miss; uint64_t get_hit_l0; @@ -11793,6 +12289,8 @@ DEF_SHOW_FUNC(block_cache_data_miss, BLOCK_CACHE_DATA_MISS) DEF_SHOW_FUNC(block_cache_data_hit, BLOCK_CACHE_DATA_HIT) DEF_SHOW_FUNC(block_cache_data_add, BLOCK_CACHE_DATA_ADD) DEF_SHOW_FUNC(bloom_filter_useful, BLOOM_FILTER_USEFUL) +DEF_SHOW_FUNC(bloom_filter_full_positive, BLOOM_FILTER_FULL_POSITIVE) +DEF_SHOW_FUNC(bloom_filter_full_true_positive, BLOOM_FILTER_FULL_TRUE_POSITIVE) DEF_SHOW_FUNC(memtable_hit, MEMTABLE_HIT) DEF_SHOW_FUNC(memtable_miss, MEMTABLE_MISS) DEF_SHOW_FUNC(get_hit_l0, GET_HIT_L0) @@ -12027,6 +12525,8 @@ static SHOW_VAR rocksdb_status_vars[] = { DEF_STATUS_VAR(block_cache_data_hit), DEF_STATUS_VAR(block_cache_data_add), DEF_STATUS_VAR(bloom_filter_useful), + DEF_STATUS_VAR(bloom_filter_full_positive), + DEF_STATUS_VAR(bloom_filter_full_true_positive), DEF_STATUS_VAR(memtable_hit), DEF_STATUS_VAR(memtable_miss), DEF_STATUS_VAR(get_hit_l0), @@ -12084,6 +12584,10 @@ static SHOW_VAR rocksdb_status_vars[] = { &rocksdb_snapshot_conflict_errors, SHOW_LONGLONG), DEF_STATUS_VAR_PTR("wal_group_syncs", &rocksdb_wal_group_syncs, SHOW_LONGLONG), + DEF_STATUS_VAR_PTR("manual_compactions_processed", + &rocksdb_manual_compactions_processed, SHOW_LONGLONG), + DEF_STATUS_VAR_PTR("manual_compactions_running", + &rocksdb_manual_compactions_running, SHOW_LONGLONG), DEF_STATUS_VAR_PTR("number_sst_entry_put", &rocksdb_num_sst_entry_put, SHOW_LONGLONG), DEF_STATUS_VAR_PTR("number_sst_entry_delete", &rocksdb_num_sst_entry_delete, @@ -12158,6 +12662,42 @@ void Rdb_background_thread::run() { } } + // Recalculate statistics for indexes. + if (rocksdb_stats_recalc_rate) { + std::unordered_map> + to_recalc; + + if (rdb_indexes_to_recalc.empty()) { + struct Rdb_index_collector : public Rdb_tables_scanner { + int add_table(Rdb_tbl_def *tdef) override { + for (uint i = 0; i < tdef->m_key_count; i++) { + rdb_indexes_to_recalc.push_back( + tdef->m_key_descr_arr[i]->get_gl_index_id()); + } + return HA_EXIT_SUCCESS; + } + } collector; + ddl_manager.scan_for_tables(&collector); + } + + while (to_recalc.size() < rocksdb_stats_recalc_rate && + !rdb_indexes_to_recalc.empty()) { + const auto index_id = rdb_indexes_to_recalc.back(); + rdb_indexes_to_recalc.pop_back(); + + std::shared_ptr keydef = + ddl_manager.safe_find(index_id); + + if (keydef) { + to_recalc.insert(std::make_pair(keydef->get_gl_index_id(), keydef)); + } + } + + if (!to_recalc.empty()) { + calculate_stats(to_recalc, false); + } + } + // Set the next timestamp for mysql_cond_timedwait() (which ends up calling // pthread_cond_timedwait()) to wait on. ts_next_sync.tv_sec = ts.tv_sec + WAKE_UP_INTERVAL; @@ -12167,12 +12707,152 @@ void Rdb_background_thread::run() { ddl_manager.persist_stats(); } -bool ha_rocksdb::check_bloom_and_set_bounds(THD *thd, const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond, - const bool use_all_keys) { +/* + A background thread to handle manual compactions, + except for dropping indexes/tables. Every second, it checks + pending manual compactions, and it calls CompactRange if there is. +*/ +void Rdb_manual_compaction_thread::run() { + mysql_mutex_init(0, &m_mc_mutex, MY_MUTEX_INIT_FAST); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + for (;;) { + if (m_stop) { + break; + } + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += 1; + + const auto ret MY_ATTRIBUTE((__unused__)) = + mysql_cond_timedwait(&m_signal_cond, &m_signal_mutex, &ts); + if (m_stop) { + break; + } + // make sure, no program error is returned + DBUG_ASSERT(ret == 0 || ret == ETIMEDOUT); + RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); + + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + // Grab the first item and proceed, if not empty. + if (m_requests.empty()) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + continue; + } + Manual_compaction_request &mcr = m_requests.begin()->second; + DBUG_ASSERT(mcr.cf != nullptr); + DBUG_ASSERT(mcr.state == Manual_compaction_request::INITED); + mcr.state = Manual_compaction_request::RUNNING; + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + + DBUG_ASSERT(mcr.state == Manual_compaction_request::RUNNING); + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s started.", mcr.mc_id, + mcr.cf->GetName().c_str()); + rocksdb_manual_compactions_running++; + if (rocksdb_debug_manual_compaction_delay > 0) { + my_sleep(rocksdb_debug_manual_compaction_delay * 1000000); + } + // CompactRange may take a very long time. On clean shutdown, + // it is cancelled by CancelAllBackgroundWork, then status is + // set to shutdownInProgress. + const rocksdb::Status s = rdb->CompactRange( + getCompactRangeOptions(mcr.concurrency), mcr.cf, mcr.start, mcr.limit); + rocksdb_manual_compactions_running--; + if (s.ok()) { + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s ended.", mcr.mc_id, + mcr.cf->GetName().c_str()); + } else { + // NO_LINT_DEBUG + sql_print_information("Manual Compaction id %d cf %s aborted. %s", + mcr.mc_id, mcr.cf->GetName().c_str(), s.getState()); + if (!s.IsShutdownInProgress()) { + rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); + } else { + DBUG_ASSERT(m_requests.size() == 1); + } + } + rocksdb_manual_compactions_processed++; + clear_manual_compaction_request(mcr.mc_id, false); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + } + clear_all_manual_compaction_requests(); + DBUG_ASSERT(m_requests.empty()); + RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); + mysql_mutex_destroy(&m_mc_mutex); +} + +void Rdb_manual_compaction_thread::clear_all_manual_compaction_requests() { + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + m_requests.clear(); + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); +} + +void Rdb_manual_compaction_thread::clear_manual_compaction_request( + int mc_id, bool init_only) { + bool erase = true; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + auto it = m_requests.find(mc_id); + if (it != m_requests.end()) { + if (init_only) { + Manual_compaction_request mcr = it->second; + if (mcr.state != Manual_compaction_request::INITED) { + erase = false; + } + } + if (erase) { + m_requests.erase(it); + } + } else { + // Current code path guarantees that erasing by the same mc_id happens + // at most once. INITED state may be erased by a thread that requested + // the compaction. RUNNING state is erased by mc thread only. + DBUG_ASSERT(0); + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); +} + +int Rdb_manual_compaction_thread::request_manual_compaction( + rocksdb::ColumnFamilyHandle *cf, rocksdb::Slice *start, + rocksdb::Slice *limit, int concurrency) { + int mc_id = -1; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + if (m_requests.size() >= rocksdb_max_manual_compactions) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return mc_id; + } + Manual_compaction_request mcr; + mc_id = mcr.mc_id = ++m_latest_mc_id; + mcr.state = Manual_compaction_request::INITED; + mcr.cf = cf; + mcr.start = start; + mcr.limit = limit; + mcr.concurrency = concurrency; + m_requests.insert(std::make_pair(mcr.mc_id, mcr)); + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return mc_id; +} + +bool Rdb_manual_compaction_thread::is_manual_compaction_finished(int mc_id) { + bool finished = false; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + if (m_requests.count(mc_id) == 0) { + finished = true; + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return finished; +} + +bool ha_rocksdb::check_bloom_and_set_bounds( + THD *thd, const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, + const bool use_all_keys, size_t bound_len, uchar *const lower_bound, + uchar *const upper_bound, rocksdb::Slice *lower_bound_slice, + rocksdb::Slice *upper_bound_slice) { bool can_use_bloom = can_use_bloom_filter(thd, kd, eq_cond, use_all_keys); if (!can_use_bloom) { - setup_iterator_bounds(kd, eq_cond); + setup_iterator_bounds(kd, eq_cond, bound_len, lower_bound, upper_bound, + lower_bound_slice, upper_bound_slice); } return can_use_bloom; } @@ -12282,7 +12962,6 @@ void rdb_update_global_stats(const operation_type &type, uint count, int rdb_get_table_perf_counters(const char *const tablename, Rdb_perf_counters *const counters) { - DBUG_ASSERT(counters != nullptr); DBUG_ASSERT(tablename != nullptr); Rdb_table_handler *table_handler; @@ -12321,9 +13000,6 @@ const char *get_rdb_io_error_string(const RDB_IO_ERROR_TYPE err_type) { // In case of core dump generation we want this function NOT to be optimized // so that we can capture as much data as possible to debug the root cause // more efficiently. -#pragma GCC push_options -#pragma GCC optimize("O0") - void rdb_handle_io_error(const rocksdb::Status status, const RDB_IO_ERROR_TYPE err_type) { if (status.IsIOError()) { @@ -12338,6 +13014,9 @@ void rdb_handle_io_error(const rocksdb::Status status, } case RDB_IO_ERROR_BG_THREAD: { rdb_log_status_error(status, "BG thread failed to write to RocksDB"); + /* NO_LINT_DEBUG */ + sql_print_error("MyRocks: aborting on BG write error."); + abort(); break; } case RDB_IO_ERROR_GENERAL: { @@ -12373,8 +13052,6 @@ void rdb_handle_io_error(const rocksdb::Status status, } } -#pragma GCC pop_options - Rdb_dict_manager *rdb_get_dict_manager(void) { return &dict_manager; } Rdb_ddl_manager *rdb_get_ddl_manager(void) { return &ddl_manager; } @@ -12672,6 +13349,42 @@ static void rocksdb_set_wal_bytes_per_sync( RDB_MUTEX_UNLOCK_CHECK(rdb_sysvars_mutex); } +/* + Validating and updating block cache size via sys_var::check path. + SetCapacity may take seconds when reducing block cache, and + sys_var::update holds LOCK_global_system_variables mutex, so + updating block cache size is done at check path instead. +*/ +static int rocksdb_validate_set_block_cache_size( + THD *thd MY_ATTRIBUTE((__unused__)), + struct st_mysql_sys_var *const var MY_ATTRIBUTE((__unused__)), + void *var_ptr, struct st_mysql_value *value) { + DBUG_ASSERT(value != nullptr); + + long long new_value; + + /* value is NULL */ + if (value->val_int(value, &new_value)) { + return HA_EXIT_FAILURE; + } + + if (new_value < RDB_MIN_BLOCK_CACHE_SIZE || + (uint64_t)new_value > (uint64_t)LONGLONG_MAX) { + return HA_EXIT_FAILURE; + } + + RDB_MUTEX_LOCK_CHECK(rdb_block_cache_resize_mutex); + const rocksdb::BlockBasedTableOptions &table_options = + rdb_get_table_options(); + + if (rocksdb_block_cache_size != new_value && table_options.block_cache) { + table_options.block_cache->SetCapacity(new_value); + } + *static_cast(var_ptr) = static_cast(new_value); + RDB_MUTEX_UNLOCK_CHECK(rdb_block_cache_resize_mutex); + return HA_EXIT_SUCCESS; +} + static int rocksdb_validate_update_cf_options(THD * /* unused */, struct st_mysql_sys_var * /*unused*/, @@ -12849,6 +13562,13 @@ double ha_rocksdb::read_time(uint index, uint ranges, ha_rows rows) { DBUG_RETURN((rows / 20.0) + 1); } +void ha_rocksdb::print_error(int error, myf errflag) { + if (error == HA_ERR_ROCKSDB_STATUS_BUSY) { + error = HA_ERR_LOCK_DEADLOCK; + } + handler::print_error(error, errflag); +} + std::string rdb_corruption_marker_file_name() { std::string ret(rocksdb_datadir); ret.append("/ROCKSDB_CORRUPTED"); @@ -12885,5 +13605,6 @@ mysql_declare_plugin(rocksdb_se){ myrocks::rdb_i_s_perf_context, myrocks::rdb_i_s_perf_context_global, myrocks::rdb_i_s_cfoptions, myrocks::rdb_i_s_compact_stats, myrocks::rdb_i_s_global_info, myrocks::rdb_i_s_ddl, - myrocks::rdb_i_s_index_file_map, myrocks::rdb_i_s_lock_info, - myrocks::rdb_i_s_trx_info, myrocks::rdb_i_s_deadlock_info mysql_declare_plugin_end; + myrocks::rdb_i_s_sst_props, myrocks::rdb_i_s_index_file_map, + myrocks::rdb_i_s_lock_info, myrocks::rdb_i_s_trx_info, + myrocks::rdb_i_s_deadlock_info mysql_declare_plugin_end; diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 0b3b473d841..ca141657413 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -104,6 +104,7 @@ struct Rdb_deadlock_info { std::string table_name; }; std::vector path; + int64_t deadlock_time; ulonglong victim_trx_id; }; @@ -144,6 +145,11 @@ const char *const BG_THREAD_NAME = "myrocks-bg"; */ const char *const INDEX_THREAD_NAME = "myrocks-index"; +/* + Name for the manual compaction thread. +*/ +const char *const MANUAL_COMPACTION_THREAD_NAME = "myrocks-mc"; + /* Separator between partition name and the qualifier. Sample usage: @@ -485,6 +491,12 @@ class ha_rocksdb : public my_core::handler { const rocksdb::Snapshot *m_scan_it_snapshot; + /* Buffers used for upper/lower bounds for m_scan_it. */ + uchar *m_scan_it_lower_bound; + uchar *m_scan_it_upper_bound; + rocksdb::Slice m_scan_it_lower_bound_slice; + rocksdb::Slice m_scan_it_upper_bound_slice; + Rdb_tbl_def *m_tbl_def; /* Primary Key encoder from KeyTupleFormat to StorageFormat */ @@ -548,12 +560,6 @@ class ha_rocksdb : public my_core::handler { uchar *m_dup_sk_packed_tuple; uchar *m_dup_sk_packed_tuple_old; - /* Buffers used for passing upper/bound eq conditions. */ - uchar *m_eq_cond_lower_bound; - uchar *m_eq_cond_upper_bound; - rocksdb::Slice m_eq_cond_lower_bound_slice; - rocksdb::Slice m_eq_cond_upper_bound_slice; - /* Temporary space for packing VARCHARs (we provide it to pack_record()/pack_index_tuple() calls). @@ -635,13 +641,20 @@ class ha_rocksdb : public my_core::handler { enum ha_rkey_function find_flag) const MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); void setup_iterator_bounds(const Rdb_key_def &kd, - const rocksdb::Slice &eq_cond); + const rocksdb::Slice &eq_cond, size_t bound_len, + uchar *const lower_bound, uchar *const upper_bound, + rocksdb::Slice *lower_bound_slice, + rocksdb::Slice *upper_bound_slice); bool can_use_bloom_filter(THD *thd, const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, const bool use_all_keys); bool check_bloom_and_set_bounds(THD *thd, const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, - const bool use_all_keys); + const bool use_all_keys, size_t bound_len, + uchar *const lower_bound, + uchar *const upper_bound, + rocksdb::Slice *lower_bound_slice, + rocksdb::Slice *upper_bound_slice); void setup_scan_iterator(const Rdb_key_def &kd, rocksdb::Slice *slice, const bool use_all_keys, const uint eq_cond_len) MY_ATTRIBUTE((__nonnull__)); @@ -834,7 +847,7 @@ public: HA_REC_NOT_IN_SEQ | HA_CAN_INDEX_BLOBS | (m_pk_can_be_decoded ? HA_PRIMARY_KEY_IN_READ_INDEX : 0) | HA_PRIMARY_KEY_REQUIRED_FOR_POSITION | HA_NULL_IN_KEY | - HA_PARTIAL_COLUMN_READ); + HA_PARTIAL_COLUMN_READ | HA_ONLINE_ANALYZE); } bool init_with_fields() override; @@ -1009,6 +1022,7 @@ public: } virtual double read_time(uint, uint, ha_rows rows) override; + virtual void print_error(int error, myf errflag) override; int open(const char *const name, int mode, uint test_if_locked) override MY_ATTRIBUTE((__warn_unused_result__)); @@ -1123,8 +1137,8 @@ private: MY_ATTRIBUTE((__nonnull__)); int compare_key_parts(const KEY *const old_key, - const KEY *const new_key) const; - MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); + const KEY *const new_key) const + MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); int compare_keys(const KEY *const old_key, const KEY *const new_key) const MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); @@ -1180,7 +1194,7 @@ private: int update_pk(const Rdb_key_def &kd, const struct update_row_info &row_info, const bool &pk_changed) MY_ATTRIBUTE((__warn_unused_result__)); int update_sk(const TABLE *const table_arg, const Rdb_key_def &kd, - const struct update_row_info &row_info) + const struct update_row_info &row_info, const bool bulk_load_sk) MY_ATTRIBUTE((__warn_unused_result__)); int update_indexes(const struct update_row_info &row_info, const bool &pk_changed) @@ -1234,7 +1248,9 @@ private: int finalize_bulk_load(bool print_client_error = true) MY_ATTRIBUTE((__warn_unused_result__)); -public: + int calculate_stats_for_table() MY_ATTRIBUTE((__warn_unused_result__)); + + public: int index_init(uint idx, bool sorted) override MY_ATTRIBUTE((__warn_unused_result__)); int index_end() override MY_ATTRIBUTE((__warn_unused_result__)); @@ -1327,9 +1343,6 @@ public: MY_ATTRIBUTE((__warn_unused_result__)); int analyze(THD *const thd, HA_CHECK_OPT *const check_opt) override MY_ATTRIBUTE((__warn_unused_result__)); - int calculate_stats(const TABLE *const table_arg, THD *const thd, - HA_CHECK_OPT *const check_opt) - MY_ATTRIBUTE((__warn_unused_result__)); enum_alter_inplace_result check_if_supported_inplace_alter( TABLE *altered_table, @@ -1356,7 +1369,7 @@ public: virtual void rpl_after_delete_rows() override; virtual void rpl_before_update_rows() override; virtual void rpl_after_update_rows() override; - virtual bool use_read_free_rpl(); + virtual bool use_read_free_rpl() override; private: /* Flags tracking if we are inside different replication operation */ diff --git a/storage/rocksdb/ha_rocksdb_proto.h b/storage/rocksdb/ha_rocksdb_proto.h index 71ec6957456..a2d014f2f01 100644 --- a/storage/rocksdb/ha_rocksdb_proto.h +++ b/storage/rocksdb/ha_rocksdb_proto.h @@ -39,7 +39,12 @@ enum RDB_IO_ERROR_TYPE { const char *get_rdb_io_error_string(const RDB_IO_ERROR_TYPE err_type); void rdb_handle_io_error(const rocksdb::Status status, - const RDB_IO_ERROR_TYPE err_type); + const RDB_IO_ERROR_TYPE err_type) +#if defined(__clang__) + MY_ATTRIBUTE((optnone)); +#else + MY_ATTRIBUTE((optimize("O0"))); +#endif int rdb_normalize_tablename(const std::string &tablename, std::string *str) MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); diff --git a/storage/rocksdb/mysql-test/rocksdb/combinations b/storage/rocksdb/mysql-test/rocksdb/combinations new file mode 100644 index 00000000000..c3f6b9d0396 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/combinations @@ -0,0 +1,6 @@ +[write_committed] +rocksdb_write_policy=write_committed + +[write_prepared] +rocksdb_write_policy=write_prepared +rocksdb_commit_time_batch_for_recovery=on diff --git a/storage/rocksdb/mysql-test/rocksdb/include/have_write_committed.inc b/storage/rocksdb/mysql-test/rocksdb/include/have_write_committed.inc new file mode 100644 index 00000000000..681b966f680 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/include/have_write_committed.inc @@ -0,0 +1,3 @@ +if (`select count(*) = 0 from information_schema.session_variables where variable_name = 'rocksdb_write_policy' and variable_value = 'write_committed';`) { + --skip Test requires write_committed policy +} diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result index b7b60f26816..b6b942c3d70 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result @@ -287,11 +287,13 @@ set global rocksdb_bulk_load=1; show global variables like 'rocksdb_bulk_load%'; Variable_name Value rocksdb_bulk_load ON +rocksdb_bulk_load_allow_sk OFF rocksdb_bulk_load_allow_unsorted OFF rocksdb_bulk_load_size 1000 show session variables like 'rocksdb_bulk_load%'; Variable_name Value rocksdb_bulk_load ON +rocksdb_bulk_load_allow_sk OFF rocksdb_bulk_load_allow_unsorted OFF rocksdb_bulk_load_size 1000 CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB; @@ -335,6 +337,7 @@ SET session rocksdb_merge_buf_size = 340; show variables like 'rocksdb_bulk_load%'; Variable_name Value rocksdb_bulk_load OFF +rocksdb_bulk_load_allow_sk OFF rocksdb_bulk_load_allow_unsorted OFF rocksdb_bulk_load_size 1000 CREATE TABLE t1 (a VARCHAR(80)) ENGINE=RocksDB; @@ -442,3 +445,24 @@ t1 CREATE TABLE `t1` ( KEY `kb` (`b`(8)) ) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin DROP TABLE t1; +SET @prior_rocksdb_table_stats_sampling_pct = @@rocksdb_table_stats_sampling_pct; +set global rocksdb_table_stats_sampling_pct = 100; +CREATE TABLE t1 (a INT, b INT, PRIMARY KEY ka(a)) ENGINE=RocksDB; +INSERT INTO t1 (a, b) VALUES (1, 10); +INSERT INTO t1 (a, b) VALUES (2, 10); +INSERT INTO t1 (a, b) VALUES (3, 20); +INSERT INTO t1 (a, b) VALUES (4, 20); +set global rocksdb_force_flush_memtable_now=1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +SHOW INDEX in t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 0 PRIMARY 1 a A 4 NULL NULL LSMTREE +ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +SHOW INDEX in t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 0 PRIMARY 1 a A 4 NULL NULL LSMTREE +t1 1 kb 1 b A 2 NULL NULL YES LSMTREE +DROP TABLE t1; +SET global rocksdb_table_stats_sampling_pct = @prior_rocksdb_table_stats_sampling_pct; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result index ec9c940f250..97310071ef2 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result @@ -15,7 +15,7 @@ count(b) 3000000 ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; -ERROR HY000: Status error 10 received from RocksDB: Operation aborted: Failed to acquire lock due to max_num_locks limit +ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to max_num_locks limit' from ROCKSDB set session rocksdb_bulk_load=1; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; set session rocksdb_bulk_load=0; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result b/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result index d86792a6469..6f811e177ba 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/allow_no_primary_key.result @@ -262,3 +262,28 @@ SELECT * FROM t1; a b 36 foo DROP TABLE t1; +# +# Issue #834/MDEV-15304 ALTER TABLE table_with_hidden_pk causes Can't +# write; duplicate key in table error and/or crash +# +CREATE TABLE t1 (a INT, KEY(a)) ENGINE=RocksDB; +INSERT INTO t1 VALUES (1),(1+1); +create table t2 (a int); +insert into t2 values (10),(20),(30); +BEGIN; +select * from t2; +a +10 +20 +30 +alter table t1 force; +select * from t1; +a +insert into t1 values (100); +select * from t1; +a +1 +2 +100 +rollback; +drop table t1,t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result index 79030e35225..8b51c126e2c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_debug.result @@ -59,12 +59,10 @@ insert into t values (); set debug="+d,crash_commit_before"; commit; ERROR HY000: Lost connection to MySQL server during query -select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; -table_schema table_name auto_increment -test t 4 -select max(i) from t; -max(i) -3 +select max(i) into @row_max from t; +select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; +table_schema table_name auto_increment > @row_max +test t 1 # After engine prepare begin; insert into t values (); @@ -72,12 +70,10 @@ insert into t values (); set debug="+d,crash_commit_after_prepare"; commit; ERROR HY000: Lost connection to MySQL server during query -select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; -table_schema table_name auto_increment -test t 4 -select max(i) from t; -max(i) -3 +select max(i) into @row_max from t; +select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; +table_schema table_name auto_increment > @row_max +test t 1 # After binlog begin; insert into t values (); @@ -85,12 +81,10 @@ insert into t values (); set debug="+d,crash_commit_after_log"; commit; ERROR HY000: Lost connection to MySQL server during query -select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; -table_schema table_name auto_increment -test t 6 -select max(i) from t; -max(i) -5 +select max(i) into @row_max from t; +select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; +table_schema table_name auto_increment > @row_max +test t 1 # After everything begin; insert into t values (); @@ -98,10 +92,8 @@ insert into t values (); set debug="+d,crash_commit_after"; commit; ERROR HY000: Lost connection to MySQL server during query -select table_schema, table_name, auto_increment from information_schema.tables where table_name = 't'; -table_schema table_name auto_increment -test t 8 -select max(i) from t; -max(i) -7 +select max(i) into @row_max from t; +select table_schema, table_name, auto_increment > @row_max from information_schema.tables where table_name = 't'; +table_schema table_name auto_increment > @row_max +test t 1 drop table t; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result index 848d362f9e1..17e6bedb882 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result @@ -141,3 +141,21 @@ SELECT * FROM t1; a b 18446744073709551613 a DROP TABLE t1; +#---------------------------------- +# Issue #792 Crash in autoincrement +#---------------------------------- +CREATE TABLE t1(C1 DOUBLE AUTO_INCREMENT KEY,C2 CHAR) ENGINE=ROCKSDB; +INSERT INTO t1 VALUES(2177,0); +DROP TABLE t1; +CREATE TABLE t0(c0 BLOB) ENGINE=ROCKSDB; +INSERT INTO t0 VALUES(0); +ALTER TABLE t0 AUTO_INCREMENT=0; +DROP TABLE t0; +#---------------------------------- +# Issue #869 Crash in autoincrement +#---------------------------------- +CREATE TABLE t1 (pk INT AUTO_INCREMENT, a INT, PRIMARY KEY(pk)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (1); +UPDATE t1 SET pk = 3; +ALTER TABLE t1 AUTO_INCREMENT 2; +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter5.result b/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter5.result new file mode 100644 index 00000000000..058d3608c75 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter5.result @@ -0,0 +1,62 @@ +# +# Issue #809: Wrong query result with bloom filters +# +create table t1 ( +id1 bigint not null, +id2 bigint not null, +id3 varchar(100) not null, +id4 int not null, +id5 int not null, +value bigint, +value2 varchar(100), +primary key (id1, id2, id3, id4) COMMENT 'rev:bf5_1' +) engine=ROCKSDB; +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t3(seq int); +insert into t3 +select +1+ A.a + B.a* 10 + C.a * 100 + D.a * 1000 +from t2 A, t2 B, t2 C, t2 D; +insert t1 +select +(seq+9) div 10, (seq+4) div 5, (seq+4) div 5, seq, seq, 1000, "aaabbbccc" +from t3; +set global rocksdb_force_flush_memtable_now=1; +# Full table scan +explain +select * from t1 limit 10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10000 NULL +select * from t1 limit 10; +id1 id2 id3 id4 id5 value value2 +1000 2000 2000 10000 10000 1000 aaabbbccc +1000 2000 2000 9999 9999 1000 aaabbbccc +1000 2000 2000 9998 9998 1000 aaabbbccc +1000 2000 2000 9997 9997 1000 aaabbbccc +1000 2000 2000 9996 9996 1000 aaabbbccc +1000 1999 1999 9995 9995 1000 aaabbbccc +1000 1999 1999 9994 9994 1000 aaabbbccc +1000 1999 1999 9993 9993 1000 aaabbbccc +1000 1999 1999 9992 9992 1000 aaabbbccc +1000 1999 1999 9991 9991 1000 aaabbbccc +# An index scan starting from the end of the table: +explain +select * from t1 order by id1 desc,id2 desc, id3 desc, id4 desc limit 1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 122 NULL 1 NULL +select * from t1 order by id1 desc,id2 desc, id3 desc, id4 desc limit 1; +id1 id2 id3 id4 id5 value value2 +1000 2000 2000 10000 10000 1000 aaabbbccc +create table t4 ( +pk int unsigned not null primary key, +kp1 int unsigned not null, +kp2 int unsigned not null, +col1 int unsigned, +key(kp1, kp2) comment 'rev:bf5_2' +) engine=rocksdb; +insert into t4 values (1, 0xFFFF, 0xFFF, 12345); +# This must not fail an assert: +select * from t4 force index(kp1) where kp1=0xFFFFFFFF and kp2<=0xFFFFFFFF order by kp2 desc; +pk kp1 kp2 col1 +drop table t1,t2,t3,t4; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter_bulk_load.result b/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter_bulk_load.result new file mode 100644 index 00000000000..4b02d1103cf --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter_bulk_load.result @@ -0,0 +1,15 @@ +create table r1 (id bigint primary key, value bigint) engine=rocksdb; +create table r2 (id bigint, value bigint, primary key (id) comment 'cf2') engine=rocksdb; +set session rocksdb_bulk_load=1; +set session rocksdb_bulk_load=0; +select variable_value into @h from information_schema.global_status where variable_name='rocksdb_block_cache_filter_hit'; +insert into r1 values (100, 100); +select variable_value-@h from information_schema.global_status where variable_name='rocksdb_block_cache_filter_hit'; +variable_value-@h +1 +select variable_value into @h from information_schema.global_status where variable_name='rocksdb_block_cache_filter_hit'; +insert into r2 values (100, 100); +select variable_value-@h from information_schema.global_status where variable_name='rocksdb_block_cache_filter_hit'; +variable_value-@h +0 +DROP TABLE r1, r2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_sk.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_sk.result new file mode 100644 index 00000000000..42f820a2a42 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_sk.result @@ -0,0 +1,229 @@ +SET rocksdb_bulk_load_size=15; +CREATE TABLE t4 (a INT, b INT, c INT, +PRIMARY KEY (a), +KEY (b), +KEY (c) COMMENT "rev:cf") ENGINE=ROCKSDB; +CREATE TABLE t3 (a INT, b INT, c INT, +PRIMARY KEY (a), +KEY (b), +KEY (c) COMMENT "rev:cf") ENGINE=ROCKSDB; +CREATE TABLE t2 (a INT, b INT, c INT, +PRIMARY KEY (a), +KEY (b), +KEY (c) COMMENT "rev:cf") ENGINE=ROCKSDB; +CREATE TABLE t1 (a INT, b INT, c INT, +PRIMARY KEY (a), +KEY (b), +KEY (c) COMMENT "rev:cf") ENGINE=ROCKSDB; +SET rocksdb_bulk_load=1; +INSERT INTO t1 SELECT * FROM t3 FORCE INDEX (PRIMARY) ORDER BY a; +SELECT count(*) FROM t1 FORCE INDEX (PRIMARY); +count(*) +0 +SELECT count(*) FROM t1 FORCE INDEX (b); +count(*) +10 +SELECT count(*) FROM t1 FORCE INDEX (c); +count(*) +10 +SET rocksdb_bulk_load=0; +SELECT * FROM t1 FORCE INDEX (PRIMARY); +a b c +-9 11 11 +-7 9 9 +-5 7 7 +-3 5 5 +-1 3 3 +2 0 0 +4 -2 -2 +6 -4 -4 +8 -6 -6 +10 -8 -8 +SELECT b FROM t1 FORCE INDEX (b); +b +-8 +-6 +-4 +-2 +0 +3 +5 +7 +9 +11 +SELECT c FROM t1 FORCE INDEX (c); +c +-8 +-6 +-4 +-2 +0 +3 +5 +7 +9 +11 +Checksums should match +CHECKSUM TABLE t3; +Table Checksum +test.t3 3862424802 +CHECKSUM TABLE t1; +Table Checksum +test.t1 3862424802 +SET rocksdb_bulk_load_allow_sk=1; +SET rocksdb_bulk_load=1; +INSERT INTO t4 SELECT * FROM t3 FORCE INDEX (PRIMARY) ORDER BY a; +SELECT count(*) FROM t4 FORCE INDEX (PRIMARY); +count(*) +0 +SELECT count(*) FROM t4 FORCE INDEX (b); +count(*) +0 +SELECT count(*) FROM t4 FORCE INDEX (c); +count(*) +0 +SET rocksdb_bulk_load=0; +SELECT * FROM t4 FORCE INDEX (PRIMARY); +a b c +-9 11 11 +-7 9 9 +-5 7 7 +-3 5 5 +-1 3 3 +2 0 0 +4 -2 -2 +6 -4 -4 +8 -6 -6 +10 -8 -8 +SELECT b FROM t4 FORCE INDEX (b); +b +-8 +-6 +-4 +-2 +0 +3 +5 +7 +9 +11 +SELECT c FROM t4 FORCE INDEX (c); +c +-8 +-6 +-4 +-2 +0 +3 +5 +7 +9 +11 +Checksums should match +CHECKSUM TABLE t3; +Table Checksum +test.t3 3862424802 +CHECKSUM TABLE t4; +Table Checksum +test.t4 3862424802 +SET rocksdb_bulk_load_allow_unsorted=1; +SET rocksdb_bulk_load_allow_sk=1; +SET rocksdb_bulk_load=1; +INSERT INTO t2 SELECT * FROM t3 WHERE b >= 0 ORDER BY b; +INSERT INTO t2 SELECT * FROM t3 WHERE b < 0 ORDER BY b; +SELECT count(*) FROM t2 FORCE INDEX (PRIMARY); +count(*) +0 +SELECT count(*) FROM t2 FORCE INDEX (b); +count(*) +0 +SELECT count(*) FROM t2 FORCE INDEX (c); +count(*) +0 +SELECT count(*) FROM t2 FORCE INDEX (PRIMARY); +count(*) +0 +SELECT count(*) FROM t2 FORCE INDEX (b); +count(*) +0 +SELECT count(*) FROM t2 FORCE INDEX (c); +count(*) +0 +SET rocksdb_bulk_load=0; +SELECT * FROM t2 FORCE INDEX (PRIMARY); +a b c +-19 21 21 +-17 19 19 +-15 17 17 +-13 15 15 +-11 13 13 +-9 11 11 +-7 9 9 +-5 7 7 +-3 5 5 +-1 3 3 +2 0 0 +4 -2 -2 +6 -4 -4 +8 -6 -6 +10 -8 -8 +12 -10 -10 +14 -12 -12 +16 -14 -14 +18 -16 -16 +20 -18 -18 +SELECT b FROM t2 FORCE INDEX (b); +b +-18 +-16 +-14 +-12 +-10 +-8 +-6 +-4 +-2 +0 +3 +5 +7 +9 +11 +13 +15 +17 +19 +21 +SELECT c FROM t2 FORCE INDEX (c); +c +-18 +-16 +-14 +-12 +-10 +-8 +-6 +-4 +-2 +0 +3 +5 +7 +9 +11 +13 +15 +17 +19 +21 +Checksums should match +CHECKSUM TABLE t3; +Table Checksum +test.t3 1495594118 +CHECKSUM TABLE t2; +Table Checksum +test.t2 1495594118 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/cardinality.result b/storage/rocksdb/mysql-test/rocksdb/r/cardinality.result index 94585af6cfc..029c68ae6b1 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/cardinality.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/cardinality.result @@ -82,4 +82,19 @@ t1 1 t1_5 2 c1 A 100000 NULL NULL YES LSMTREE SELECT table_name, table_rows FROM information_schema.tables WHERE table_schema = DATABASE(); table_name table_rows t1 100000 -drop table t1; +CREATE TABLE t2 (a INT, b INT, c INT, d INT, e INT, f INT, g INT, +PRIMARY KEY (a), KEY (c, b, a, d, e, f, g)) +ENGINE=ROCKSDB; +SET GLOBAL rocksdb_force_flush_memtable_now = 1; +ANALYZE TABLE t2; +Table Op Msg_type Msg_text +test.t2 analyze status OK +cardinality of the columns after 'a' must be equal to the cardinality of column 'a' +SELECT CARDINALITY INTO @c FROM information_schema.statistics WHERE TABLE_NAME='t2' AND INDEX_NAME='c' AND COLUMN_NAME='a'; +SELECT COLUMN_NAME, CARDINALITY = @c FROM information_schema.statistics WHERE TABLE_NAME='t2' AND INDEX_NAME='c' AND SEQ_IN_INDEX > 3; +COLUMN_NAME CARDINALITY = @c +d 1 +e 1 +f 1 +g 1 +drop table t1, t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/collation.result b/storage/rocksdb/mysql-test/rocksdb/r/collation.result index edd3fd1f187..b4354ad353f 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/collation.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/collation.result @@ -1,6 +1,7 @@ -SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS; -DROP TABLE IF EXISTS t1; +call mtr.add_suppression("Invalid pattern"); CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; +ALTER TABLE t1 ADD INDEX (value); +ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). DROP TABLE t1; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; ERROR HY000: Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). @@ -13,6 +14,7 @@ SET GLOBAL rocksdb_strict_collation_check=1; CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value2)) engine=rocksdb charset utf8; DROP TABLE t1; CREATE TABLE t1 (id varchar(20), value varchar(50), value2 varchar(50), value3 text, primary key (id), index(value, value2)) engine=rocksdb charset latin1 collate latin1_bin; +ALTER TABLE t1 collate=latin1_general_ci; DROP TABLE t1; CREATE TABLE t1 (id varchar(20), value varchar(50), value2 varchar(50), value3 text, primary key (id), index(value, value2)) engine=rocksdb charset utf8 collate utf8_bin; DROP TABLE t1; @@ -126,4 +128,16 @@ CREATE TABLE abcd (id INT PRIMARY KEY, value varchar(50), index(value)) engine=r ERROR HY000: Unsupported collation on string indexed column test.abcd.value Use binary collation (binary, latin1_bin, utf8_bin). DROP TABLE abc; SET GLOBAL rocksdb_strict_collation_exceptions=null; -SET GLOBAL rocksdb_strict_collation_exceptions=@start_global_value; +SET GLOBAL rocksdb_strict_collation_check=1; +CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text, index(value)) engine=rocksdb charset utf8; +Warnings: +Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). +DROP TABLE t1; +CREATE TABLE t1 (id INT primary key, value varchar(50), value2 varbinary(50), value3 text) engine=rocksdb charset utf8; +ALTER TABLE t1 ADD INDEX (value); +Warnings: +Warning 1210 Unsupported collation on string indexed column test.t1.value Use binary collation (binary, latin1_bin, utf8_bin). +DROP TABLE t1; +CREATE TABLE t1 (id varchar(20), value varchar(50), value2 varchar(50), value3 text, primary key (id), index(value, value2)) engine=rocksdb charset latin1 collate latin1_bin; +ALTER TABLE t1 collate=latin1_general_ci; +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/com_rpc_tx.result b/storage/rocksdb/mysql-test/rocksdb/r/com_rpc_tx.result new file mode 100644 index 00000000000..789ce12e900 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/com_rpc_tx.result @@ -0,0 +1,21 @@ +CREATE DATABASE db_rpc; +USE db_rpc; +CREATE TABLE t1(pk INT PRIMARY KEY) ENGINE=rocksdb; +SET GLOBAL rocksdb_enable_2pc=1; +SET autocommit = 0; +SET autocommit = 0; +BEGIN; +BEGIN; +SELECT * from t1; +pk +SELECT * from t1; +pk +INSERT INTO t1 VALUES(1); +INSERT INTO t1 VALUES(2); +COMMIT; +COMMIT; +SELECT * from db_rpc.t1; +pk +1 +2 +DROP DATABASE db_rpc; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/create_no_primary_key_table.result b/storage/rocksdb/mysql-test/rocksdb/r/create_no_primary_key_table.result new file mode 100644 index 00000000000..e5aeb57ebdf --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/create_no_primary_key_table.result @@ -0,0 +1,38 @@ +USE mysql; +CREATE TABLE mysql_table (a INT) ENGINE=ROCKSDB; +CREATE TABLE test.mysql_table (a INT) ENGINE=ROCKSDB; +ERROR HY000: Table without primary key cannot be created outside mysql schema. +USE test; +CREATE TABLE mysql_table (a INT) ENGINE=ROCKSDB; +ERROR HY000: Table without primary key cannot be created outside mysql schema. +CREATE TABLE IF NOT EXISTS mysql_table_2 (a INT) ENGINE=ROCKSDB; +ERROR HY000: Table without primary key cannot be created outside mysql schema. +CREATE TABLE mysql_table_no_cols ENGINE=ROCKSDB; +ERROR HY000: Table without primary key cannot be created outside mysql schema. +CREATE TABLE mysql.mysql_table_2 (a INT) ENGINE=ROCKSDB; +CREATE TABLE mysql_primkey (a INT PRIMARY KEY, b INT, c INT, d INT, INDEX (c)) ENGINE=ROCKSDB; +ALTER TABLE mysql_primkey DROP b, DROP a, ADD (f INT PRIMARY KEY); +ALTER TABLE mysql_primkey DROP PRIMARY KEY; +ERROR HY000: Table without primary key cannot be created outside mysql schema. +CREATE TABLE mysql_primkey2 (a INT PRIMARY KEY, b INT, c INT) ENGINE=ROCKSDB; +ALTER TABLE mysql_primkey2 DROP b; +ALTER TABLE mysql_primkey2 ADD (b INT); +ALTER TABLE mysql_primkey2 DROP c, DROP A; +ERROR HY000: Table without primary key cannot be created outside mysql schema. +CREATE TABLE mysql_primkey3 (a INT PRIMARY KEY, b INT, c INT, INDEX indexonb (b), INDEX indexonc (c)) ENGINE=ROCKSDB; +ALTER TABLE mysql_primkey3 DROP INDEX indexonb; +ALTER TABLE mysql_primkey3 DROP c; +ALTER TABLE mysql_primkey3 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +CREATE TABLE mysql_primkey4(a INT, b INT, PRIMARY KEY(a), INDEX si (a, b)) ENGINE=ROCKSDB; +DROP INDEX si ON mysql_primkey4; +DROP INDEX `PRIMARY` ON mysql_primkey4; +ERROR HY000: Table without primary key cannot be created outside mysql schema. +ALTER TABLE mysql.mysql_table ADD PRIMARY KEY (a); +ALTER TABLE mysql.mysql_table DROP PRIMARY KEY; +DROP TABLE mysql_primkey; +DROP TABLE mysql_primkey2; +DROP TABLE mysql_primkey3; +DROP TABLE mysql_primkey4; +USE mysql; +DROP TABLE mysql_table; +DROP TABLE mysql_table_2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/ddl_high_priority.result b/storage/rocksdb/mysql-test/rocksdb/r/ddl_high_priority.result index 39130475349..50733f81598 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/ddl_high_priority.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/ddl_high_priority.result @@ -45,7 +45,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 alter table t1 modify i bigint;; set high_priority_ddl = 0; @@ -98,7 +98,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 alter table t1 rename t1_new;; set high_priority_ddl = 0; @@ -152,7 +152,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 drop table t1;; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 @@ -202,7 +202,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 drop table t1;; set high_priority_ddl = 0; @@ -251,7 +251,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 alter table t1 modify i bigint;; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 @@ -302,7 +302,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 create index idx1 on t1 (i);; set high_priority_ddl = 0; @@ -342,7 +342,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 drop index idx1 on t1;; set high_priority_ddl = 0; @@ -390,7 +390,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 truncate t1;; set high_priority_ddl = 0; @@ -438,7 +438,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 create trigger ins_sum before insert on t1 for each row set @sum = @sum + new.i;; set high_priority_ddl = 0; @@ -478,7 +478,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 drop trigger ins_sum;; set high_priority_ddl = 0; @@ -528,7 +528,7 @@ set high_priority_ddl = 1; select @@high_priority_ddl; @@high_priority_ddl 1 -lock tables t1 write; +rename table t1 to t2; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on table metadata: test.t1 optimize table t1;; Table Op Msg_type Msg_text @@ -538,6 +538,55 @@ connection: default (for show processlist) show processlist; Id User Host db Command Time State Info Rows examined Rows sent Tid Srv_Id root test