From 81f94c26a4fb60345e260ba25f39315f3d9a6f3b Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 24 May 2021 16:48:27 +0200 Subject: [PATCH 01/30] MDEV-15730: rename --stream=xbstream to --stream=mbstream mbstream is already supported as a format name after MDEV-24580, but additional code refactoring has been done to correctly display the format name in log files and to check if the mbstream utility is in the path. Also, for xtrabackup-v2 (only available in the 10.2) both utilities are supported - both xbstram and mbstream, since they are interchangeable in this context. In this case, the original innobackupex always receives the correct --stream=xbstream option as input, but the user can actually try to use the mbstream utility during the transfer (if the user explicitly specifies this in the configuration file). --- scripts/wsrep_sst_common.sh | 2 +- scripts/wsrep_sst_mariabackup.sh | 24 +++++++++++--------- scripts/wsrep_sst_xtrabackup-v2.sh | 35 +++++++++++++++++++----------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 952a37f75d2..ceab62b4c56 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -295,7 +295,7 @@ case "$1" in value="$1" fi fi - if [ $option == 'h' ]; then + if [ $option = 'h' ]; then if [ -z "$WSREP_SST_OPT_DATA" ]; then MYSQLD_OPT_DATADIR="${value%/}" fi diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 5618c704dbc..530d3e49256 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -55,7 +55,7 @@ ib_home_dir="" ib_log_dir="" ib_undo_dir="" -sfmt="tar" +sfmt="" strmcmd="" tfmt="" tcmd="" @@ -97,7 +97,6 @@ if [ ! -x "$MARIABACKUP_BIN" ]; then wsrep_log_error 'mariabackup binary not found in $PATH' exit 42 fi -MBSTREAM_BIN=mbstream DATA="$WSREP_SST_OPT_DATA" INFO_FILE="xtrabackup_galera_info" @@ -481,21 +480,26 @@ read_cnf() get_stream() { if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then - wsrep_log_info "Streaming with ${sfmt}" + sfmt='mbstream' + MBSTREAM_BIN="$(command -v mbstream)" + if [ -z "$MBSTREAM_BIN" ]; then + wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path" + exit 42 + fi if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - strmcmd="$MBSTREAM_BIN -x" + strmcmd="'$MBSTREAM_BIN' -x" else - strmcmd="$MBSTREAM_BIN -c '$INFO_FILE'" + strmcmd="'$MBSTREAM_BIN' -c '$INFO_FILE'" fi else - sfmt="tar" - wsrep_log_info "Streaming with tar" - if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]]; then - strmcmd="tar xfi -" + sfmt='tar' + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + strmcmd='tar xfi -' else strmcmd="tar cf - '$INFO_FILE'" fi fi + wsrep_log_info "Streaming with $sfmt" } get_proc() @@ -930,7 +934,7 @@ setup_commands() fi INNOAPPLY="$MARIABACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY" INNOMOVE="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" - INNOBACKUP="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream='$sfmt' --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP" + INNOBACKUP="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP" } get_stream diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 2eef6c661a5..b0d5c0aedf1 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -48,7 +48,7 @@ ib_home_dir="" ib_log_dir="" ib_undo_dir="" -sfmt="tar" +sfmt="" strmcmd="" tfmt="" tcmd="" @@ -489,22 +489,28 @@ read_cnf() get_stream() { - if [[ $sfmt == 'xbstream' ]];then - wsrep_log_info "Streaming with xbstream" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - strmcmd="xbstream -x" + if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then + XBSTREAM_BIN=$(command -v "$sfmt") + if [ -z "$XBSTREAM_BIN" ]; then + if [ -z "$XBSTREAM_BIN" ]; then + wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path" + exit 42 + fi + fi + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + strmcmd="'$XBSTREAM_BIN' -x" else - strmcmd="xbstream -c '${INFO_FILE}'" + strmcmd="'$XBSTREAM_BIN' -c '$INFO_FILE'" fi else - sfmt="tar" - wsrep_log_info "Streaming with tar" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - strmcmd="tar xfi -" + sfmt='tar' + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + strmcmd='tar xfi -' else - strmcmd="tar cf - '${INFO_FILE}'" + strmcmd="tar cf - '$INFO_FILE'" fi fi + wsrep_log_info "Streaming with $sfmt" } get_proc() @@ -575,7 +581,6 @@ cleanup_donor() wsrep_log_error "xtrabackup process is still running. Killing... " kill_xtrabackup fi - fi rm -f "${DATA}/${IST_FILE}" || true @@ -852,7 +857,11 @@ cd "$OLD_PWD" setup_commands () { INNOAPPLY="$INNOBACKUPEX_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY" INNOMOVE="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $impts --move-back --force-non-empty-directories '$DATA' $INNOMOVE" - INNOBACKUP="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream='$sfmt' '$itmpdir' $INNOBACKUP" + sfmt_work="$sfmt" + if [ "$sfmt" = 'mbstream' ]; then + sfmt_work='xbstream' + fi + INNOBACKUP="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP" } if [ "$WSREP_SST_OPT_ROLE" = "donor" ] From fe7e44d8ad5d7fe9c91f476353a3e1749f18afc6 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 25 May 2021 05:08:25 +0200 Subject: [PATCH 02/30] MDEV-21192: SST failing when enabling IPV6 The following features have been added: 1) Automatic addition of the pf = ip6 option for socat when it can be recognized by the format of the connection address; 2) Automatically add or remove extra commas at the beginning and at the end of sockopt, for example, sockopt='pf=ip6' and sockopt=',pf=ip6' work equally well; Also, due to interference in the code of the get_transfer() function, I also refactored it and now: 3) encrypt = 4 is supported not only for xtrabackup-v2, but also for mariabackup - this can help with migration from Percona; 4) Improved setting of 'commonname' option for encrypt=3 and encrypt=4 modes; --- .../t/galera_ipv6_mariabackup.cnf | 1 - .../t/galera_ipv6_mariabackup_section.cnf | 1 + .../galera_3nodes/t/galera_ipv6_mysqldump.cnf | 3 - .../galera_3nodes/t/galera_ipv6_rsync.cnf | 3 - .../t/galera_ipv6_rsync_section.cnf | 3 - scripts/wsrep_sst_common.sh | 22 ++ scripts/wsrep_sst_mariabackup.sh | 235 ++++++++++-------- scripts/wsrep_sst_rsync.sh | 22 +- scripts/wsrep_sst_xtrabackup-v2.sh | 208 ++++++++-------- 9 files changed, 265 insertions(+), 233 deletions(-) diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.cnf b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.cnf index 8432b1c368c..3b0f0173b35 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.cnf +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.cnf @@ -34,4 +34,3 @@ bind-address=:: [SST] transferfmt=@ENV.MTR_GALERA_TFMT streamfmt=xbstream -sockopt=",pf=ip6" diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.cnf b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.cnf index aa3da690416..a3ab9a85707 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.cnf +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.cnf @@ -38,4 +38,5 @@ bind-address=:: [SST] transferfmt=@ENV.MTR_GALERA_TFMT streamfmt=xbstream +# Not needed, but left here for debugging: sockopt=",pf=ip6" diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mysqldump.cnf b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mysqldump.cnf index 5e77a45210b..c163f798335 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mysqldump.cnf +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mysqldump.cnf @@ -24,6 +24,3 @@ wsrep_provider_options='base_host=[::1];base_port=@mysqld.3.#galera_port;gmcast. wsrep_sst_receive_address='[::1]:@mysqld.3.#sst_port' wsrep_node_incoming_address='[::1]:@mysqld.3.port' bind-address=:: - -[SST] -sockopt=",pf=ip6" diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync.cnf b/mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync.cnf index 5e77a45210b..c163f798335 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync.cnf +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync.cnf @@ -24,6 +24,3 @@ wsrep_provider_options='base_host=[::1];base_port=@mysqld.3.#galera_port;gmcast. wsrep_sst_receive_address='[::1]:@mysqld.3.#sst_port' wsrep_node_incoming_address='[::1]:@mysqld.3.port' bind-address=:: - -[SST] -sockopt=",pf=ip6" diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync_section.cnf b/mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync_section.cnf index 809b83bb782..ee92efeff1d 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync_section.cnf +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync_section.cnf @@ -29,6 +29,3 @@ wsrep_provider_options='base_host=[::1];base_port=@mysqld.3.#galera_port;gmcast. wsrep_sst_receive_address='[::1]:@mysqld.3.#sst_port' wsrep_node_incoming_address='[::1]:@mysqld.3.port' bind-address=:: - -[SST] -sockopt=",pf=ip6" diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index ceab62b4c56..261912b86ad 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1021,3 +1021,25 @@ check_for_version() [ $z1 -lt $z2 ] && return 1 return 0 } + +trim_string() +{ + if [ -n "$BASH_VERSION" ]; then + local pattern="[![:space:]${2:-}]" + local x="${1#*$pattern}" + local z=${#1} + x=${#x} + if [ $x -ne $z ]; then + local y="${1%$pattern*}" + y=${#y} + x=$(( $z-$x-1 )) + y=$(( $y-$x+1 )) + printf '%s' "${1:$x:$y}" + else + printf '' + fi + else + local pattern="[[:space:]${2:-}]" + echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" + fi +} diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 530d3e49256..0fdc765ff8b 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -34,8 +34,6 @@ ecode=0 ssyslog="" ssystag="" MARIABACKUP_PID="" -SST_PORT="" -REMOTEIP="" tcert="" tpem="" tkey="" @@ -94,7 +92,7 @@ declare -a RC MARIABACKUP_BIN="$(command -v mariabackup)" if [ ! -x "$MARIABACKUP_BIN" ]; then - wsrep_log_error 'mariabackup binary not found in $PATH' + wsrep_log_error 'mariabackup binary not found in path' exit 42 fi @@ -214,8 +212,6 @@ get_keys() get_transfer() { - TSST_PORT="$SST_PORT" - if [ $tfmt = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc @@ -237,7 +233,7 @@ get_transfer() wsrep_log_info "Using traditional netcat as streamer" tcmd="$tcmd -l -p" fi - tcmd="$tcmd $TSST_PORT" + tcmd="$tcmd $SST_PORT" else # Check to see if netcat supports the '-N' flag. # -N Shutdown the network socket after EOF on stdin @@ -259,7 +255,7 @@ get_transfer() wsrep_log_info "Using traditional netcat as streamer" tcmd="$tcmd -q0" fi - tcmd="$tcmd $WSREP_SST_OPT_HOST_UNESCAPED $TSST_PORT" + tcmd="$tcmd $WSREP_SST_OPT_HOST_UNESCAPED $SST_PORT" fi else tfmt='socat' @@ -267,8 +263,38 @@ get_transfer() wsrep_log_info "Using socat as streamer" wsrep_check_programs socat - if [ $encrypt -eq 2 -o $encrypt -eq 3 ] && ! socat -V | grep -q -F 'WITH_OPENSSL 1'; then - wsrep_log_error "Encryption requested, but socat is not OpenSSL enabled (encrypt=$encrypt)" + if [ -n "$sockopt" ]; then + sockopt=$(trim_string "$sockopt" ',') + if [ -n "$sockopt" ]; then + sockopt=",$sockopt" + fi + fi + + # Add an option for ipv6 if needed: + if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then + # If sockopt contains 'pf=ip6' somewhere in the middle, + # this will not interfere with socat, but exclude the trivial + # cases when sockopt contains 'pf=ip6' as prefix or suffix: + if [ "$sockopt" = "${sockopt#,pf=ip6}" -a \ + "$sockopt" = "${sockopt%,pf=ip6}" ] + then + sockopt=",pf=ip6$sockopt" + fi + fi + + if [ $encrypt -lt 2 ]; then + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + tcmd="socat -u TCP-LISTEN:$SST_PORT,reuseaddr$sockopt stdio" + else + tcmd="socat -u stdio TCP:$REMOTEIP:$SST_PORT$sockopt" + fi + return + fi + + if ! socat -V | grep -q -F 'WITH_OPENSSL 1'; then + wsrep_log_error "******** FATAL ERROR ************************************************ " + wsrep_log_error "* Encryption requested, but socat is not OpenSSL enabled (encrypt=$encrypt) *" + wsrep_log_error "********************************************************************* " exit 2 fi @@ -281,11 +307,21 @@ get_transfer() exit 2 fi - if ! check_for_version "$SOCAT_VERSION" "1.7.3"; then + local action='Decrypting' + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr" + else + tcmd="socat -u stdio openssl-connect:$REMOTEIP:$SST_PORT" + action='Encrypting' + fi + + if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then # socat versions < 1.7.3 will have 512-bit dhparams (too small) # so create 2048-bit dhparams and send that as a parameter: check_for_dhparams - sockopt=",dhparam='$ssl_dhparams'$sockopt" + if [ -n "$ssl_dhparams" ]; then + tcmd="$tcmd,dhparam='$ssl_dhparams'" + fi fi if [ $encrypt -eq 2 ]; then @@ -294,15 +330,10 @@ get_transfer() wsrep_log_error "Both PEM and CRT files required" exit 22 fi + tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt" stagemsg="$stagemsg-OpenSSL-Encrypted-2" - if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - wsrep_log_info "Decrypting with cert=${tpem}, cafile=${tcert}" - tcmd="socat -u openssl-listen:$TSST_PORT,reuseaddr,cert='$tpem',cafile='$tcert'$sockopt stdio" - else - wsrep_log_info "Encrypting with cert=${tpem}, cafile=${tcert}" - tcmd="socat -u stdio openssl-connect:$REMOTEIP:$TSST_PORT,cert='$tpem',cafile='$tcert'$sockopt" - fi - elif [ $encrypt -eq 3 ]; then + wsrep_log_info "$action with cert=$tpem, cafile=$tcert" + elif [ $encrypt -eq 3 -o $encrypt -eq 4 ]; then wsrep_log_info "Using openssl based encryption with socat: with key and crt" if [ -z "$tpem" -o -z "$tkey" ]; then wsrep_log_error "Both certificate and key files required" @@ -310,36 +341,34 @@ get_transfer() fi stagemsg="$stagemsg-OpenSSL-Encrypted-3" if [ -z "$tcert" ]; then - # no verification - if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, verify=0" - tcmd="socat -u openssl-listen:$TSST_PORT,reuseaddr,cert='$tpem',key='$tkey',verify=0$sockopt stdio" - else - wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, verify=0" - tcmd="socat -u stdio openssl-connect:$REMOTEIP:$TSST_PORT,cert='$tpem',key='$tkey',verify=0$sockopt" + if [ $encrypt -eq 4 ]; then + wsrep_log_error "Peer certificate required if encrypt=4" + exit 22 fi + # no verification + tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0$sockopt" + wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0" else # CA verification - if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}" - tcmd="socat -u openssl-listen:$TSST_PORT,reuseaddr,cert='$tpem',key='$tkey',cafile='$tcert'$sockopt stdio" + if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then + CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'" + elif [ $encrypt -eq 4 ]; then + CN_option=",commonname=''" + elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then + CN_option=',commonname=localhost' else - CN_option="" - if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then - CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'" - elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CN_option=',commonname=localhost' - fi - wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}" - tcmd="socat -u stdio openssl-connect:$REMOTEIP:$TSST_PORT,cert='$tpem',key='$tkey',cafile='$tcert'$CN_option$sockopt" + CN_option=",commonname='$WSREP_SST_OPT_HOST_UNSECAPED'" fi + tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'$CN_option$sockopt" + wsrep_log_info "$action with cert=$tpem, key=$tkey, cafile=$tcert" fi else - if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - tcmd="socat -u TCP-LISTEN:$TSST_PORT,reuseaddr$sockopt stdio" - else - tcmd="socat -u stdio TCP:$REMOTEIP:$TSST_PORT$sockopt" - fi + wsrep_log_info "Unknown encryption mode: encrypt=$encrypt" + exit 22 + fi + + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + tcmd="$tcmd stdio" fi fi } @@ -347,7 +376,7 @@ get_transfer() get_footprint() { pushd "$WSREP_SST_OPT_DATA" 1>/dev/null - payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c | awk 'END { print $1 }') + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }') if [ "$compress" != 'none' ]; then # QuickLZ has around 50% compression ratio # When compression/compaction used, the progress is only an approximate. @@ -440,7 +469,7 @@ read_cnf() sockopt=$(parse_cnf sst sockopt "") progress=$(parse_cnf sst progress "") ttime=$(parse_cnf sst time 0) - cpat='.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$' + cpat='.*\.pem$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$' [ "$OS" = 'FreeBSD' ] && cpat=$(echo "$cpat" | sed 's/\\|/|/g') cpat=$(parse_cnf sst cpat "$cpat") scomp=$(parse_cnf sst compressor "") @@ -807,8 +836,6 @@ monitor_process() done } -wsrep_check_programs "$MARIABACKUP_BIN" - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then @@ -842,7 +869,6 @@ INNODB_DATA_HOME_DIR=$(pwd -P) cd "$OLD_PWD" if [ $ssyslog -eq 1 ]; then - if [ -n "$(command -v logger)" ]; then wsrep_log_info "Logging all stderr of SST/mariabackup to syslog" @@ -860,70 +886,65 @@ if [ $ssyslog -eq 1 ]; then else wsrep_log_error "logger not in path: $PATH. Ignoring" fi - INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply" INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move" INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" - else - -if [ $sstlogarchive -eq 1 ] -then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") - - if [ -n "$sstlogarchivedir" ]; then - if [ ! -d "$sstlogarchivedir" ]; then - mkdir -p "$sstlogarchivedir" - fi - fi - - if [ -e "$INNOAPPLYLOG" ] + if [ $sstlogarchive -eq 1 ] then - if [ -n "$sstlogarchivedir" ] - then - newfile=$(basename "$INNOAPPLYLOG") - newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" - else - newfile="$INNOAPPLYLOG.$ARCHIVETIMESTAMP" - fi - wsrep_log_info "Moving '$INNOAPPLYLOG' to '$newfile'" - mv "$INNOAPPLYLOG" "$newfile" - gzip "$newfile" - fi + ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") - if [ -e "$INNOMOVELOG" ] - then - if [ -n "$sstlogarchivedir" ] - then - newfile=$(basename "$INNOMOVELOG") - newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" - else - newfile="$INNOMOVELOG.$ARCHIVETIMESTAMP" + if [ -n "$sstlogarchivedir" ]; then + if [ ! -d "$sstlogarchivedir" ]; then + mkdir -p "$sstlogarchivedir" + fi fi - wsrep_log_info "Moving '$INNOMOVELOG' to '$newfile'" - mv "$INNOMOVELOG" "$newfile" - gzip "$newfile" - fi - if [ -e "$INNOBACKUPLOG" ] - then - if [ -n "$sstlogarchivedir" ] + if [ -e "$INNOAPPLYLOG" ] then - newfile=$(basename "$INNOBACKUPLOG") - newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" - else - newfile="$INNOBACKUPLOG.$ARCHIVETIMESTAMP" + if [ -n "$sstlogarchivedir" ] + then + newfile=$(basename "$INNOAPPLYLOG") + newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" + else + newfile="$INNOAPPLYLOG.$ARCHIVETIMESTAMP" + fi + wsrep_log_info "Moving '$INNOAPPLYLOG' to '$newfile'" + mv "$INNOAPPLYLOG" "$newfile" + gzip "$newfile" fi - wsrep_log_info "Moving '$INNOBACKUPLOG' to '$newfile'" - mv "$INNOBACKUPLOG" "$newfile" - gzip "$newfile" - fi -fi + if [ -e "$INNOMOVELOG" ] + then + if [ -n "$sstlogarchivedir" ] + then + newfile=$(basename "$INNOMOVELOG") + newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" + else + newfile="$INNOMOVELOG.$ARCHIVETIMESTAMP" + fi + wsrep_log_info "Moving '$INNOMOVELOG' to '$newfile'" + mv "$INNOMOVELOG" "$newfile" + gzip "$newfile" + fi + + if [ -e "$INNOBACKUPLOG" ] + then + if [ -n "$sstlogarchivedir" ] + then + newfile=$(basename "$INNOBACKUPLOG") + newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" + else + newfile="$INNOBACKUPLOG.$ARCHIVETIMESTAMP" + fi + wsrep_log_info "Moving '$INNOBACKUPLOG' to '$newfile'" + mv "$INNOBACKUPLOG" "$newfile" + gzip "$newfile" + fi + fi INNOAPPLY="&> '$INNOAPPLYLOG'" INNOMOVE="&> '$INNOMOVELOG'" INNOBACKUP="2> '$INNOBACKUPLOG'" - fi setup_commands() @@ -1001,9 +1022,9 @@ then send_donor "$DATA" "$stagemsg-gtid" + # Restore the transport commmand to its original state tcmd="$ttcmd" - # Restore the transport commmand to its original state if [ -n "$progress" ]; then get_footprint tcmd="$pcmd | $tcmd" @@ -1015,7 +1036,7 @@ then wsrep_log_info "Sleeping before data transfer for SST" sleep 10 - wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP}:${SST_PORT}" + wsrep_log_info "Streaming the backup to joiner at $REMOTEIP:$SST_PORT" # Add compression to the head of the stream (if specified) if [ -n "$scomp" ]; then @@ -1030,8 +1051,8 @@ then iopts="$iopts --databases-exclude='lost+found'" if [ ${FORCE_FTWRL:-0} -eq 1 ]; then - wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" - iopts="$iopts --no-backup-locks" + wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" + iopts="$iopts --no-backup-locks" fi # if compression is enabled for backup files, then add the @@ -1052,8 +1073,8 @@ then set -e if [ ${RC[0]} -ne 0 ]; then - wsrep_log_error "${MARIABACKUP_BIN} finished with error: ${RC[0]}. " \ - "Check syslog or ${INNOBACKUPLOG} for details" + wsrep_log_error "mariabackup finished with error: ${RC[0]}. " \ + "Check syslog or '$INNOBACKUPLOG' for details" exit 22 elif [ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]; then wsrep_log_error "$tcmd finished with error: ${RC[1]}" @@ -1185,7 +1206,7 @@ then then if [ -d "$DATA/.sst" ]; then - wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing" + wsrep_log_info "WARNING: Stale temporary SST directory: '$DATA/.sst' from previous state transfer. Removing" rm -rf "$DATA/.sst" fi mkdir -p "$DATA/.sst" @@ -1300,21 +1321,21 @@ then timeit "mariabackup prepare stage" "$INNOAPPLY" if [ $? -ne 0 ]; then - wsrep_log_error "${MARIABACKUP_BIN} apply finished with errors. Check syslog or ${INNOAPPLYLOG} for details" + wsrep_log_error "mariabackup apply finished with errors. Check syslog or '$INNOAPPLYLOG' for details" exit 22 fi MAGIC_FILE="$TDATA/$INFO_FILE" + wsrep_log_info "Moving the backup to ${TDATA}" timeit "mariabackup move stage" "$INNOMOVE" - if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing ${DATA}" rm -rf "$DATA" DATA="$TDATA" else wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" - wsrep_log_error "Check syslog or ${INNOMOVELOG} for details" + wsrep_log_error "Check syslog or '$INNOMOVELOG' for details" exit 22 fi diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index f954046382c..0cc13ee4191 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -261,16 +261,20 @@ then ;; 'VERIFY_CA') VERIFY_OPT='verifyChain = yes' - # check if the address is an ip-address (v4 or v6): - if echo "$WSREP_SST_OPT_HOST_UNESCAPED" | \ - grep -q -E '^([0-9]+(\.[0-9]+){3,3}|[0-9a-fA-F]?(\:[0-9a-fA-F]*)+)$' - then - CHECK_OPT="checkIP = $WSREP_SST_OPT_HOST_UNESCAPED" + if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then + CHECK_OPT="checkHost = $WSREP_SST_OPT_REMOTE_USER" else - CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" - fi - if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CHECK_OPT_LOCAL="checkHost = localhost" + # check if the address is an ip-address (v4 or v6): + if echo "$WSREP_SST_OPT_HOST_UNESCAPED" | \ + grep -q -E '^([0-9]+(\.[0-9]+){3,3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$' + then + CHECK_OPT="checkIP = $WSREP_SST_OPT_HOST_UNESCAPED" + else + CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" + fi + if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then + CHECK_OPT_LOCAL="checkHost = localhost" + fi fi ;; *) diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index b0d5c0aedf1..b5ede2b29cb 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -32,8 +32,8 @@ ecode=0 ssyslog="" ssystag="" XTRABACKUP_PID="" -tca="" tcert="" +tpem="" tkey="" sockopt="" progress="" @@ -224,20 +224,9 @@ verify_file_exists() get_transfer() { - TSST_PORT="$WSREP_SST_OPT_PORT" - - if [[ $tfmt == 'nc' ]];then + if [ $tfmt = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - - if [[ $encrypt -eq 2 || $encrypt -eq 3 || $encrypt -eq 4 ]]; then - wsrep_log_error "******** FATAL ERROR *********************** " - wsrep_log_error "* Using SSL encryption (encrypt= 2, 3, or 4) " - wsrep_log_error "* is not supported when using nc(netcat). " - wsrep_log_error "******************************************** " - exit 22 - fi - tcmd="nc" if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then if nc -h 2>&1 | grep -q 'ncat'; then @@ -256,7 +245,7 @@ get_transfer() wsrep_log_info "Using traditional netcat as streamer" tcmd="$tcmd -l -p" fi - tcmd="$tcmd $TSST_PORT" + tcmd="$tcmd $SST_PORT" else # Check to see if netcat supports the '-N' flag. # -N Shutdown the network socket after EOF on stdin @@ -278,113 +267,118 @@ get_transfer() wsrep_log_info "Using traditional netcat as streamer" tcmd="$tcmd -q0" fi - tcmd="$tcmd $WSREP_SST_OPT_HOST_UNESCAPED $TSST_PORT" + tcmd="$tcmd $WSREP_SST_OPT_HOST_UNESCAPED $SST_PORT" fi else tfmt='socat' + wsrep_log_info "Using socat as streamer" wsrep_check_programs socat - donor_extra="" - joiner_extra="" - if [[ $encrypt -eq 2 || $encrypt -eq 3 || $encrypt -eq 4 ]]; then - if ! socat -V | grep -q WITH_OPENSSL; then - wsrep_log_error "******** FATAL ERROR ****************** " - wsrep_log_error "* socat is not openssl enabled. " - wsrep_log_error "* Unable to encrypt SST communications. " - wsrep_log_error "*************************************** " - exit 2 - fi - - # Determine the socat version - SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') - if [ -z "$SOCAT_VERSION" ]; then - wsrep_log_error "******** FATAL ERROR ******************" - wsrep_log_error "* Cannot determine the socat version. *" - wsrep_log_error "***************************************" - exit 2 - fi - if ! check_for_version "$SOCAT_VERSION" "1.7.3"; then - # socat versions < 1.7.3 will have 512-bit dhparams (too small) - # so create 2048-bit dhparams and send that as a parameter: - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then - # dhparams check (will create ssl_dhparams if needed) - check_for_dhparams - joiner_extra=",dhparam='$ssl_dhparams'" - fi - else - # socat version >= 1.7.3, checks to see if the peername matches - # the hostname, then set commonname="" to disable the peername - # checks: - donor_extra=',commonname=""' + if [ -n "$sockopt" ]; then + sockopt=$(trim_string "$sockopt" ',') + if [ -n "$sockopt" ]; then + sockopt=",$sockopt" fi fi - if [[ $encrypt -eq 2 ]]; then - wsrep_log_warning "**** WARNING **** encrypt=2 is deprecated and will be removed in a future release" - wsrep_log_info "Using openssl based encryption with socat: with crt and ca" - - verify_file_exists "$tcert" "Both certificate and CA files are required." \ - "Please check the 'tcert' option. " - verify_file_exists "$tca" "Both certificate and CA files are required." \ - "Please check the 'tca' option. " - - stagemsg+="-OpenSSL-Encrypted-2" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - wsrep_log_info "Decrypting with CERT: $tcert, CA: $tca" - tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert='${tcert}',cafile='${tca}'${joiner_extra}${sockopt} stdio" - else - wsrep_log_info "Encrypting with CERT: $tcert, CA: $tca" - tcmd="socat -u stdio openssl-connect:${WSREP_SST_OPT_HOST}:${TSST_PORT},cert='${tcert}',cafile='${tca}'${donor_extra}${sockopt}" + # Add an option for ipv6 if needed: + if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then + # If sockopt contains 'pf=ip6' somewhere in the middle, + # this will not interfere with socat, but exclude the trivial + # cases when sockopt contains 'pf=ip6' as prefix or suffix: + if [ "$sockopt" = "${sockopt#,pf=ip6}" -a \ + "$sockopt" = "${sockopt%,pf=ip6}" ] + then + sockopt=",pf=ip6$sockopt" fi - elif [[ $encrypt -eq 3 ]];then - wsrep_log_warning "**** WARNING **** encrypt=3 is deprecated and will be removed in a future release" + fi + + if [ $encrypt -lt 2 ]; then + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + tcmd="socat -u TCP-LISTEN:$SST_PORT,reuseaddr$sockopt stdio" + else + tcmd="socat -u stdio TCP:$REMOTEIP:$SST_PORT$sockopt" + fi + return + fi + + if ! socat -V | grep -q -F 'WITH_OPENSSL 1'; then + wsrep_log_error "******** FATAL ERROR ************************************************ " + wsrep_log_error "* Encryption requested, but socat is not OpenSSL enabled (encrypt=$encrypt) *" + wsrep_log_error "********************************************************************* " + exit 2 + fi + + # Determine the socat version + SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') + if [ -z "$SOCAT_VERSION" ]; then + wsrep_log_error "******** FATAL ERROR ******************" + wsrep_log_error "* Cannot determine the socat version. *" + wsrep_log_error "***************************************" + exit 2 + fi + + local action='Decrypting' + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr" + else + tcmd="socat -u stdio openssl-connect:$REMOTEIP:$SST_PORT" + action='Encrypting' + fi + + if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then + # socat versions < 1.7.3 will have 512-bit dhparams (too small) + # so create 2048-bit dhparams and send that as a parameter: + check_for_dhparams + tcmd="$tcmd,dhparam='$ssl_dhparams'" + fi + + if [ $encrypt -eq 2 ]; then + wsrep_log_info "Using openssl based encryption with socat: with crt and pem" + if [ -z "$tpem" -o -z "$tcert" ]; then + wsrep_log_error "Both PEM and CRT files required" + exit 22 + fi + tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt" + stagemsg="$stagemsg-OpenSSL-Encrypted-2" + wsrep_log_info "$action with cert=$tpem, cafile=$tcert" + elif [ $encrypt -eq 3 -o $encrypt -eq 4 ]; then wsrep_log_info "Using openssl based encryption with socat: with key and crt" - - verify_file_exists "$tcert" "Both certificate and key files are required." \ - "Please check the 'tcert' option. " - verify_file_exists "$tkey" "Both certificate and key files are required." \ - "Please check the 'tkey' option. " - - stagemsg+="-OpenSSL-Encrypted-3" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - wsrep_log_info "Decrypting with CERT: $tcert, KEY: $tkey" - tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert='${tcert}',key='${tkey}',verify=0${joiner_extra}${sockopt} stdio" - else - wsrep_log_info "Encrypting with CERT: $tcert, KEY: $tkey" - tcmd="socat -u stdio openssl-connect:${WSREP_SST_OPT_HOST}:${TSST_PORT},cert='${tcert}',key='${tkey}',verify=0${sockopt}" + if [ -z "$tpem" -o -z "$tkey" ]; then + wsrep_log_error "Both certificate and key files required" + exit 22 fi - elif [[ $encrypt -eq 4 ]]; then - wsrep_log_info "Using openssl based encryption with socat: with key, crt, and ca" - - verify_file_exists "$ssl_ca" "CA, certificate, and key files are required." \ - "Please check the 'ssl-ca' option. " - verify_file_exists "$ssl_cert" "CA, certificate, and key files are required." \ - "Please check the 'ssl-cert' option. " - verify_file_exists "$ssl_key" "CA, certificate, and key files are required." \ - "Please check the 'ssl-key' option. " - - # Check to see that the key matches the cert - verify_cert_matches_key $ssl_cert $ssl_key - - stagemsg+="-OpenSSL-Encrypted-4" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then - wsrep_log_info "Decrypting with CERT: $ssl_cert, KEY: $ssl_key, CA: $ssl_ca" - tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert='${ssl_cert}',key='${ssl_key}',cafile='${ssl_ca}',verify=1${joiner_extra}${sockopt} stdio" + stagemsg="$stagemsg-OpenSSL-Encrypted-3" + if [ -z "$tcert" ]; then + if [ $encrypt -eq 4 ]; then + wsrep_log_error "Peer certificate required if encrypt=4" + exit 22 + fi + # no verification + tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0$sockopt" + wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0" else - wsrep_log_info "Encrypting with CERT: $ssl_cert, KEY: $ssl_key, CA: $ssl_ca" - tcmd="socat -u stdio openssl-connect:${WSREP_SST_OPT_HOST}:${TSST_PORT},cert='${ssl_cert}',key='${ssl_key}',cafile='${ssl_ca}',verify=1${donor_extra}${sockopt}" + # CA verification + if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then + CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'" + elif [ $encrypt -eq 4 ]; then + CN_option=",commonname=''" + elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then + CN_option=',commonname=localhost' + else + CN_option=",commonname='$WSREP_SST_OPT_HOST_UNSECAPED'" + fi + tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'$CN_option$sockopt" + wsrep_log_info "$action with cert=$tpem, key=$tkey, cafile=$tcert" fi else - if [[ $encrypt -eq 1 ]]; then - wsrep_log_warning "**** WARNING **** encrypt=1 is deprecated and will be removed in a future release" - fi + wsrep_log_info "Unknown encryption mode: encrypt=$encrypt" + exit 22 + fi - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then - tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" - else - tcmd="socat -u stdio TCP:${WSREP_SST_OPT_HOST}:${TSST_PORT}${sockopt}" - fi + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + tcmd="$tcmd stdio" fi fi } @@ -435,8 +429,8 @@ read_cnf() { sfmt=$(parse_cnf sst streamfmt "xbstream") tfmt=$(parse_cnf sst transferfmt "socat") - tca=$(parse_cnf sst tca "") - tcert=$(parse_cnf sst tcert "") + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") tkey=$(parse_cnf sst tkey "") encrypt=$(parse_cnf sst encrypt 0) sockopt=$(parse_cnf sst sockopt "") From d8fa71a089aa5cbe569e7f8b6fa67326a9ecab2b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 19 May 2021 22:26:02 +0200 Subject: [PATCH 03/30] MDEV-25730: maria.repair test fails with valgrind cherry-pick commit: 1fff2398ef3dda1a7e8404f18e4e165823bd4e0a MDEV-22530 post push fixes from 10.6. Followup. If the KILL happens - report it as a failure, don't eat it up silently. Note that this has to be done after `table_name` is populated, so that the error message could show it. --- sql/sql_admin.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 21afa1154f6..3fa00b81f2e 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -499,8 +499,6 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, DBUG_PRINT("admin", ("table: '%s'.'%s'", table->db, table->table_name)); DEBUG_SYNC(thd, "admin_command_kill_before_modify"); - if (thd->is_killed()) - break; strxmov(table_name, db, ".", table->table_name, NullS); thd->open_options|= extra_open_options; table->lock_type= lock_type; @@ -515,6 +513,13 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, : lock_type >= TL_WRITE_ALLOW_WRITE ? MDL_SHARED_WRITE : MDL_SHARED_READ); + if (thd->check_killed()) + { + fatal_error= true; + result_code= HA_ADMIN_FAILED; + goto send_result; + } + /* open only one table from local list of command */ while (1) { From e212415690c78b1decc8d7b7bfdb715426c4e49f Mon Sep 17 00:00:00 2001 From: sjaakola Date: Mon, 24 May 2021 18:58:57 +0300 Subject: [PATCH 04/30] MDEV-25551 applying crash with tables without PK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The underlying problem with MDEV-25551 turned out to be that transactions having changes for tables with no primary key, were not safe to apply in parallel. This is due to excessive locking in innodb side, and even non related row modifications could end up in lock conflict during applying. The fix for MDEV-25551 has disabled parallel applying for tables with no PK. This fix depends on change for wsrep-lib, where a separate PR allows application to modify transaction flags in wsrep-lib. This commit has also separate mtr test for verifying that transactions modifying a table with no primary key, will not apply in parallel. This test is a modified version of initial test created by Gabor Orosz, the reporterr of MDEV-25551. Another mtr test was added in galera_sr suite, for testing if modifying tables with no primary key would causes issues for streaming replication use cases. Reviewed-by: Jan Lindström --- .../suite/galera/r/galera_nonPK_and_PA.result | 63 +++++++ .../suite/galera/t/galera_nonPK_and_PA.test | 168 ++++++++++++++++++ .../galera_sr/r/galera_sr_nonPK_and_PA.result | 46 +++++ .../galera_sr/t/galera_sr_nonPK_and_PA.test | 109 ++++++++++++ sql/handler.cc | 30 +++- wsrep-lib | 2 +- 6 files changed, 415 insertions(+), 3 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_nonPK_and_PA.result create mode 100644 mysql-test/suite/galera/t/galera_nonPK_and_PA.test create mode 100644 mysql-test/suite/galera_sr/r/galera_sr_nonPK_and_PA.result create mode 100644 mysql-test/suite/galera_sr/t/galera_sr_nonPK_and_PA.test diff --git a/mysql-test/suite/galera/r/galera_nonPK_and_PA.result b/mysql-test/suite/galera/r/galera_nonPK_and_PA.result new file mode 100644 index 00000000000..5ad55417fd1 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_nonPK_and_PA.result @@ -0,0 +1,63 @@ +connection node_2; +connection node_1; +CREATE TABLE t1 (f1 VARCHAR(32) NOT NULL) ENGINE=InnoDB; +INSERT INTO t1 (f1) VALUES ('0e66c5227a8a'); +INSERT INTO t1 (f1) VALUES ('c6c112992c9'); +CREATE TABLE t2 (i int primary key); +connection node_2; +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_slave_threads = 2; +*************************************************************** +scenario 1, conflicting UPDATE +*************************************************************** +SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_slave_enter_sync'; +connection node_1; +START TRANSACTION; +UPDATE t1 SET f1='5ffceebfada' WHERE t1.f1 = 'c6c112992c9'; +COMMIT; +connection node_2; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +connection node_1; +START TRANSACTION; +UPDATE t1 SET f1='4ffceebfcdc' WHERE t1.f1 = '0e66c5227a8a'; +COMMIT; +connection node_2; +distance +1 +SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +*************************************************************** +scenario 2, conflicting DELETE +*************************************************************** +SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_slave_enter_sync'; +connection node_1; +START TRANSACTION; +INSERT INTO t2 VALUES (1); +DELETE FROM t1 WHERE f1='5ffceebfada'; +COMMIT; +connection node_2; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +connection node_1; +START TRANSACTION; +INSERT INTO t2 VALUES (2); +DELETE FROM t1 WHERE f1='4ffceebfcdc'; +COMMIT; +connection node_2; +distance +1 +SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; +SET GLOBAL wsrep_slave_threads = DEFAULT; +DROP TABLE t1; +DROP TABLE t2; +connection node_2; +SET GLOBAL wsrep_slave_threads = DEFAULT; diff --git a/mysql-test/suite/galera/t/galera_nonPK_and_PA.test b/mysql-test/suite/galera/t/galera_nonPK_and_PA.test new file mode 100644 index 00000000000..8a5173f576b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_nonPK_and_PA.test @@ -0,0 +1,168 @@ +# +# This test is a modified version of Gabor Orosz (GOro) test in jira tracker: +# https://jira.mariadb.org/browse/MDEV-25551 +# +# The underlying problem with MDEV-25551 turned out to be that +# transactions having changes for tables with no primary key, +# were not safe to apply in parallel. This is due to excessive locking +# in innodb side, and even non related row modifications could end up +# in lock conflict during applying. +# +# The test creates a table with no primary key definition and executes two +# transactions (in node1) modifying separate rows in the table. In node2 +# first applier is paused before commit phase, and second transaction is +# then submitted to see if it can interfere with the first transaciton. +# The fix for MDEV-25551 has disabled parallel applying for tables with no PK, +# and in the test applying of the send trasnaction should not even start, before +# the fisrt trkansaction is released from the sync point. +# The test also verifies that certification depedency status reflects the fact +# that the two transactions depend on each other. +# +# The test has two scenarios where both UPDATE and DELETE statements are verified +# to disable parallel applying +# + +--source include/galera_cluster.inc +--source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc + + +# Setup + +CREATE TABLE t1 (f1 VARCHAR(32) NOT NULL) ENGINE=InnoDB; +INSERT INTO t1 (f1) VALUES ('0e66c5227a8a'); +INSERT INTO t1 (f1) VALUES ('c6c112992c9'); + +CREATE TABLE t2 (i int primary key); + +--connection node_2 +SET SESSION wsrep_sync_wait = 0; +--let $wait_condition = SELECT COUNT(*)=2 FROM t1; +--source include/wait_condition.inc + +# Ensure that we have enough applier threads to process transactions in parallel +SET GLOBAL wsrep_slave_threads = 2; + +--echo *************************************************************** +--echo scenario 1, conflicting UPDATE +--echo *************************************************************** + +# Set up a synchronization point to catch the first transaction +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_1 +# Invoke the first transaction +START TRANSACTION; +UPDATE t1 SET f1='5ffceebfada' WHERE t1.f1 = 'c6c112992c9'; +COMMIT; + +--connection node_2 +# Wait for the first transaction to apply until commit phase +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_wait_sync_point.inc + +# remember status for received replication counter and certification dependency distance +--let $expected_wsrep_received = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_received'` +--let $cert_deps_distance = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cert_deps_distance'` + +--connection node_1 +# Invoke the second transaction +START TRANSACTION; +UPDATE t1 SET f1='4ffceebfcdc' WHERE t1.f1 = '0e66c5227a8a'; +COMMIT; + +# sleep is probably obsolete here, but it is good to give the latter update time to +# proceed in applying in node 2. In buggy version the update will start applying +# and cause conflict there. +--sleep 5 + +--connection node_2 +# Wait for the second transaction to appear in repliaction queue +--let $wait_condition = SELECT VARIABLE_VALUE= $expected_wsrep_received FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_received'; +--source include/wait_condition.inc + +# verify that certification dependency distance has dropped +--disable_query_log +--eval SELECT VARIABLE_VALUE < $cert_deps_distance as 'distance' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cert_deps_distance' +--enable_query_log + +# if deps distance dropped, it is indirect evidence that parallel applying was not approved + +# Let the first transaction to proceed +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc + +# second applier should now hit sync point +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_wait_sync_point.inc +--source include/galera_signal_sync_point.inc +--source include/galera_clear_sync_point.inc + + +--echo *************************************************************** +--echo scenario 2, conflicting DELETE +--echo *************************************************************** + +# Set up a synchronization point to catch the first transaction +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_1 +# Invoke the first transaction, mix this with insert to table having PK +START TRANSACTION; +INSERT INTO t2 VALUES (1); +DELETE FROM t1 WHERE f1='5ffceebfada'; +COMMIT; + +--connection node_2 +# Wait for the first transaction to apply until commit phase +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_wait_sync_point.inc + +# remember status for received replication counter and certification dependency distance +--let $expected_wsrep_received = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_received'` +--let $cert_deps_distance = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cert_deps_distance'` + +--connection node_1 +# Invoke the second transaction, again mix this with insert to table having PK +START TRANSACTION; +INSERT INTO t2 VALUES (2); +DELETE FROM t1 WHERE f1='4ffceebfcdc'; +COMMIT; + +# sleep is probably obsolete here, but it is good to give the latter update time to +# proceed in applying in node 2. In buggy version the update will start applying +# and cause conflict there. +--sleep 5 + +--connection node_2 +# Wait for the second transaction to appear in repliaction queue +--let $wait_condition = SELECT VARIABLE_VALUE= $expected_wsrep_received FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_received'; +--source include/wait_condition.inc + +# verify that certification dependency distance has dropped +--disable_query_log +--eval SELECT VARIABLE_VALUE < $cert_deps_distance as 'distance' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cert_deps_distance' +--enable_query_log + +# if deps distance dropped, it is indirect evidence that parallel applying was not approved + +# Let the first transaction to proceed +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc + +# second applier should now hit sync point +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_wait_sync_point.inc +--source include/galera_signal_sync_point.inc +--source include/galera_clear_sync_point.inc + +# Teardown +--connection node_1 +SET GLOBAL wsrep_slave_threads = DEFAULT; + +DROP TABLE t1; +DROP TABLE t2; +--connection node_2 +SET GLOBAL wsrep_slave_threads = DEFAULT; diff --git a/mysql-test/suite/galera_sr/r/galera_sr_nonPK_and_PA.result b/mysql-test/suite/galera_sr/r/galera_sr_nonPK_and_PA.result new file mode 100644 index 00000000000..b7e9cf49f95 --- /dev/null +++ b/mysql-test/suite/galera_sr/r/galera_sr_nonPK_and_PA.result @@ -0,0 +1,46 @@ +connection node_2; +connection node_1; +connection node_2; +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_slave_threads = 2; +flush status; +connection node_1; +CREATE TABLE t1 (f1 int, f2 int) ENGINE=InnoDB; +CREATE TABLE t2 (f1 int primary key, f2 int) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1,0); +INSERT INTO t1 VALUES (2,0); +INSERT INTO t2 VALUES (1,0); +INSERT INTO t2 VALUES (2,0); +connection node_2; +connection node_1; +set session wsrep_trx_fragment_size=1; +START TRANSACTION; +UPDATE t1 SET f2=1 where f1=1; +connection node_2; +distance +1 +SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_slave_enter_sync'; +connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connection node_1a; +update t2 set f2=1 where f1=1; +connection node_2; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_1; +UPDATE t2 set f2=2 where f1=2; +connection node_2; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; +COMMIT; +connection node_1; +SET GLOBAL wsrep_slave_threads = DEFAULT; +DROP TABLE t1; +DROP TABLE t2; +connection node_2; +SET GLOBAL wsrep_slave_threads = DEFAULT; diff --git a/mysql-test/suite/galera_sr/t/galera_sr_nonPK_and_PA.test b/mysql-test/suite/galera_sr/t/galera_sr_nonPK_and_PA.test new file mode 100644 index 00000000000..c343cd202bf --- /dev/null +++ b/mysql-test/suite/galera_sr/t/galera_sr_nonPK_and_PA.test @@ -0,0 +1,109 @@ +# +# This test is a modified version of Gabor Orosz (GOro) test in jira tracker: +# https://jira.mariadb.org/browse/MDEV-25551 +# +# The underlying problem with MDEV-25551 turned out to be that +# transactions having changes for tables with no primary key, +# were not safe to apply in parallel. This is due to excessive locking +# in innodb side, and even non related row modifications could end up +# in lock conflict during applying. +# +# The test verifies that a transaction executing a streaming replication +# will disable parallel applying if it modifies a table with no primary key. +# And, if PA was disabled temporarily, it will be relaxed if next fragment +# contains changes for table with primary key. +# + +--source include/galera_cluster.inc +--source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc + + +# Setup +--connection node_2 +SET SESSION wsrep_sync_wait = 0; + +# Ensure that we have enough applier threads to process transactions in parallel +SET GLOBAL wsrep_slave_threads = 2; + +flush status; + +--connection node_1 +CREATE TABLE t1 (f1 int, f2 int) ENGINE=InnoDB; +CREATE TABLE t2 (f1 int primary key, f2 int) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1,0); +INSERT INTO t1 VALUES (2,0); + +INSERT INTO t2 VALUES (1,0); +INSERT INTO t2 VALUES (2,0); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*)=2 FROM t2; +--source include/wait_condition.inc + +# remember status for received replication counter and certification dependency distance +--let $cert_deps_distance = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cert_deps_distance'` + +--connection node_1 +# Invoke the first transaction +set session wsrep_trx_fragment_size=1; +START TRANSACTION; +UPDATE t1 SET f2=1 where f1=1; + +--connection node_2 +# verify that certification dependency distance has dropped +--disable_query_log +--eval SELECT VARIABLE_VALUE < $cert_deps_distance as 'distance' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cert_deps_distance' +--enable_query_log + +# if deps distance dropped, it is indirect evidence that parallel applying was not approved + +# Try next that PA retricting is relaxed, if next fragment updates table t1 with primary key +# wsrep_cert_deps_distance cannot be trsuted in this test phase, we verify parallel applying +# by setting sync point for applier thread + +# Set up a synchronization point to catch update on t2 +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +--connection node_1a +update t2 set f2=1 where f1=1; + +--connection node_2 +# Wait for the update t2 to apply until commit phase +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_wait_sync_point.inc + +# Set up a synchronization point to catch the SR trx applying +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_1 +# continue SR transaction, and now update t2, which has PK +UPDATE t2 set f2=2 where f1=2; + +--connection node_2 +# Wait for the update t2 to apply until commit phase +--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_slave_enter_sync +--source include/galera_wait_sync_point.inc + +# Let the first transaction to proceed +--let $galera_sync_point = commit_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc +--source include/galera_clear_sync_point.inc +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc +--source include/galera_clear_sync_point.inc + +--connection node_1 +COMMIT; + +# Teardown +--connection node_1 +SET GLOBAL wsrep_slave_threads = DEFAULT; + +DROP TABLE t1; +DROP TABLE t2; +--connection node_2 +SET GLOBAL wsrep_slave_threads = DEFAULT; diff --git a/sql/handler.cc b/sql/handler.cc index ac5b43249db..99ec0f01320 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6808,8 +6808,21 @@ int handler::ha_update_row(const uchar *old_data, const uchar *new_data) rows_changed++; error= binlog_log_row(table, old_data, new_data, log_func); #ifdef WITH_WSREP + THD *thd= ha_thd(); + bool is_wsrep= WSREP(thd); + /* for SR, the followin wsrep_after_row() may replicate a fragment, so we have to + declare potential PA unsafe before that*/ + if (table->s->primary_key == MAX_KEY && + is_wsrep && wsrep_thd_is_local(thd)) + { + WSREP_DEBUG("marking trx as PA unsafe pk %d", table->s->primary_key); + if (thd->wsrep_cs().mark_transaction_pa_unsafe()) + { + WSREP_DEBUG("session does not have active transaction, can not mark as PA unsafe"); + } + } if (table_share->tmp_table == NO_TMP_TABLE && - WSREP(ha_thd()) && (error= wsrep_after_row(ha_thd()))) + is_wsrep && (error= wsrep_after_row(thd))) { return error; } @@ -6870,8 +6883,21 @@ int handler::ha_delete_row(const uchar *buf) rows_changed++; error= binlog_log_row(table, buf, 0, log_func); #ifdef WITH_WSREP + THD *thd= ha_thd(); + bool is_wsrep= WSREP(thd); + /* for SR, the followin wsrep_after_row() may replicate a fragment, so we have to + declare potential PA unsafe before that*/ + if (table->s->primary_key == MAX_KEY && + is_wsrep && wsrep_thd_is_local(thd)) + { + WSREP_DEBUG("marking trx as PA unsafe pk %d", table->s->primary_key); + if (thd->wsrep_cs().mark_transaction_pa_unsafe()) + { + WSREP_DEBUG("session does not have active transaction, can not mark as PA unsafe"); + } + } if (table_share->tmp_table == NO_TMP_TABLE && - WSREP(ha_thd()) && (error= wsrep_after_row(ha_thd()))) + is_wsrep && (error= wsrep_after_row(thd))) { return error; } diff --git a/wsrep-lib b/wsrep-lib index f271ad0c6e3..85b81503214 160000 --- a/wsrep-lib +++ b/wsrep-lib @@ -1 +1 @@ -Subproject commit f271ad0c6e3c647df83c1d5ec9cd26d77cef2337 +Subproject commit 85b815032145ef5c18b7a8931d84becf6df8cd18 From 88ce7cf744e624d2ea3c105ccc280325b6169339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 26 May 2021 09:06:32 +0300 Subject: [PATCH 05/30] MDEV-25769 : Galera test failure on galera_sr.GCF-627 Add wait_condition to wait until streaming log is empty. --- mysql-test/suite/galera_sr/r/GCF-627.result | 10 ++++------ mysql-test/suite/galera_sr/t/GCF-627.test | 9 +++++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/galera_sr/r/GCF-627.result b/mysql-test/suite/galera_sr/r/GCF-627.result index 891cf4af5a9..65d8c95ad08 100644 --- a/mysql-test/suite/galera_sr/r/GCF-627.result +++ b/mysql-test/suite/galera_sr/r/GCF-627.result @@ -16,11 +16,9 @@ connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; INSERT INTO t1 VALUES (2); ERROR 42S02: Table 'test.t1' doesn't exist connection node_1; -SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log; -COUNT(*) = 0 -1 +SELECT * FROM mysql.wsrep_streaming_log; +node_uuid trx_id seqno flags frag connection node_2; -SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log; -COUNT(*) = 0 -1 +SELECT * FROM mysql.wsrep_streaming_log; +node_uuid trx_id seqno flags frag DROP TABLE t2; diff --git a/mysql-test/suite/galera_sr/t/GCF-627.test b/mysql-test/suite/galera_sr/t/GCF-627.test index 86637ad8e7f..ad351eb9da6 100644 --- a/mysql-test/suite/galera_sr/t/GCF-627.test +++ b/mysql-test/suite/galera_sr/t/GCF-627.test @@ -22,9 +22,14 @@ COMMIT; INSERT INTO t1 VALUES (2); --connection node_1 -SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log; +--let $wait_condition = SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log +--source include/wait_condition.inc + +SELECT * FROM mysql.wsrep_streaming_log; --connection node_2 -SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log; +--let $wait_condition = SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log +--source include/wait_condition.inc +SELECT * FROM mysql.wsrep_streaming_log; DROP TABLE t2; From 26f9ff0a6006e2c473623a54612cbcbf77e02058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 26 May 2021 11:11:27 +0300 Subject: [PATCH 06/30] Remove unnecessary test case --- mysql-test/suite/galera/r/MDEV-25562.result | 10 ---------- mysql-test/suite/galera/t/MDEV-25562.test | 12 ------------ 2 files changed, 22 deletions(-) diff --git a/mysql-test/suite/galera/r/MDEV-25562.result b/mysql-test/suite/galera/r/MDEV-25562.result index b0d77af374b..ab8340af8a9 100644 --- a/mysql-test/suite/galera/r/MDEV-25562.result +++ b/mysql-test/suite/galera/r/MDEV-25562.result @@ -4,13 +4,3 @@ SET SESSION WSREP_ON=0; FLUSH TABLES WITH READ LOCK AND DISABLE CHECKPOINT; SET SESSION WSREP_ON=1; UNLOCK TABLES; -SET GLOBAL wsrep_ignore_apply_errors=1; -CREATE TABLE t1 (a CHAR(1)) engine=innodb; -CREATE TABLE t1 (a CHAR(1)) engine=innodb; -ERROR 42S01: Table 't1' already exists -SHOW PROCEDURE STATUS WHERE db = 'test'; -Db Name Type Definer Modified Created Security_type Comment character_set_client collation_connection Database Collation -SET GLOBAL read_only=1; -SET GLOBAL wsrep_ignore_apply_errors=DEFAULT; -SET GLOBAL read_only=DEFAULT; -DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MDEV-25562.test b/mysql-test/suite/galera/t/MDEV-25562.test index 01729936b08..b4552643089 100644 --- a/mysql-test/suite/galera/t/MDEV-25562.test +++ b/mysql-test/suite/galera/t/MDEV-25562.test @@ -9,15 +9,3 @@ SET SESSION WSREP_ON=0; FLUSH TABLES WITH READ LOCK AND DISABLE CHECKPOINT; SET SESSION WSREP_ON=1; UNLOCK TABLES; - -SET GLOBAL wsrep_ignore_apply_errors=1; -CREATE TABLE t1 (a CHAR(1)) engine=innodb; ---error ER_TABLE_EXISTS_ERROR -CREATE TABLE t1 (a CHAR(1)) engine=innodb; -SHOW PROCEDURE STATUS WHERE db = 'test'; -SET GLOBAL read_only=1; - -SET GLOBAL wsrep_ignore_apply_errors=DEFAULT; -SET GLOBAL read_only=DEFAULT; -DROP TABLE t1; - From 288b8016961e390ca71dd4657efdfcec388f1494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 26 May 2021 11:54:29 +0300 Subject: [PATCH 07/30] Fix MDEV-25562 test case. --- mysql-test/suite/galera/disabled.def | 1 - mysql-test/suite/galera/r/MDEV-25562.result | 10 ---------- mysql-test/suite/galera/t/MDEV-25562.test | 12 ------------ 3 files changed, 23 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index e5573bd4409..ec35dacb03e 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -49,4 +49,3 @@ versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest galera_wsrep_provider_unset_set: wsrep_provider is read-only for security reasons pxc-421: wsrep_provider is read-only for security reasons -MDEV-25562: MDEV-25562 FIXME: lock wait timeout exceeded diff --git a/mysql-test/suite/galera/r/MDEV-25562.result b/mysql-test/suite/galera/r/MDEV-25562.result index b0d77af374b..ab8340af8a9 100644 --- a/mysql-test/suite/galera/r/MDEV-25562.result +++ b/mysql-test/suite/galera/r/MDEV-25562.result @@ -4,13 +4,3 @@ SET SESSION WSREP_ON=0; FLUSH TABLES WITH READ LOCK AND DISABLE CHECKPOINT; SET SESSION WSREP_ON=1; UNLOCK TABLES; -SET GLOBAL wsrep_ignore_apply_errors=1; -CREATE TABLE t1 (a CHAR(1)) engine=innodb; -CREATE TABLE t1 (a CHAR(1)) engine=innodb; -ERROR 42S01: Table 't1' already exists -SHOW PROCEDURE STATUS WHERE db = 'test'; -Db Name Type Definer Modified Created Security_type Comment character_set_client collation_connection Database Collation -SET GLOBAL read_only=1; -SET GLOBAL wsrep_ignore_apply_errors=DEFAULT; -SET GLOBAL read_only=DEFAULT; -DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MDEV-25562.test b/mysql-test/suite/galera/t/MDEV-25562.test index 01729936b08..b4552643089 100644 --- a/mysql-test/suite/galera/t/MDEV-25562.test +++ b/mysql-test/suite/galera/t/MDEV-25562.test @@ -9,15 +9,3 @@ SET SESSION WSREP_ON=0; FLUSH TABLES WITH READ LOCK AND DISABLE CHECKPOINT; SET SESSION WSREP_ON=1; UNLOCK TABLES; - -SET GLOBAL wsrep_ignore_apply_errors=1; -CREATE TABLE t1 (a CHAR(1)) engine=innodb; ---error ER_TABLE_EXISTS_ERROR -CREATE TABLE t1 (a CHAR(1)) engine=innodb; -SHOW PROCEDURE STATUS WHERE db = 'test'; -SET GLOBAL read_only=1; - -SET GLOBAL wsrep_ignore_apply_errors=DEFAULT; -SET GLOBAL read_only=DEFAULT; -DROP TABLE t1; - From 17106c984b9649641068ddf108dcffd2c7773212 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Thu, 11 Mar 2021 18:17:06 +0200 Subject: [PATCH 08/30] Add Pull Request template file to the MariaDB/server repository --- .github/pull_request_template.md | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000000..6fe365495cc --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,50 @@ + + + +- [x] *The Jira issue number for this PR is: MDEV-_____* + + +## Description +TODO: fill description here + +## How can this PR be tested? +TODO: fill steps to reproduce here, if applicable, + or remove the section + + +## Basing the PR against the correct MariaDB version +- [ ] *This is a new feature and the PR is based against the latest MariaDB development branch* +- [ ] *This is a bug fix and the PR is based against the earliest branch in which the bug can be reproduced* + + +## Backward compatibility +TODO: fill details here, if applicable, or remove the section + From aa284e02374c2abdb9ae660c232c076276d2ce98 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 26 May 2021 14:35:23 +0300 Subject: [PATCH 09/30] MDEV-17749 Kill during LOCK TABLE ; ALTER TABLE causes assert The problem was that when LOCK TABLES where unwinded as part of a killed connection, unlink_all_closed_tables() did not like that there was uncommited transactions. Fixed by doing a rollback of any open transaction in this particular case. --- mysql-test/main/lock_kill.result | 36 +++++++++++++++++++ mysql-test/main/lock_kill.test | 59 ++++++++++++++++++++++++++++++++ sql/sql_base.cc | 10 ++++++ 3 files changed, 105 insertions(+) create mode 100644 mysql-test/main/lock_kill.result create mode 100644 mysql-test/main/lock_kill.test diff --git a/mysql-test/main/lock_kill.result b/mysql-test/main/lock_kill.result new file mode 100644 index 00000000000..727dd66da7f --- /dev/null +++ b/mysql-test/main/lock_kill.result @@ -0,0 +1,36 @@ +# +# MDEV-17749 Kill during LOCK TABLE ; ALTER TABLE causes assert +# +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +connect con1,localhost,root,,test; +LOCK TABLE t1 WRITE; +ALTER TABLE t1 ADD COLUMN b INT; +connection default; +Killing connection +connection con1; +connection default; +disconnect con1; +DROP TABLE t1; +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +CREATE TABLE t2 (a INT) ENGINE=InnoDB; +connect con1,localhost,root,,test; +LOCK TABLE t1 WRITE, t2 WRITE; +ALTER TABLE t1 ADD COLUMN b INT; +connection default; +Killing connection +connection con1; +connection default; +disconnect con1; +DROP TABLE t1, t2; +CREATE TABLE t1 (id int(11)) ENGINE=InnoDB; +LOCK TABLES t1 WRITE; +SET max_statement_time= 0.0001; +CREATE TRIGGER tr16 AFTER UPDATE ON t1 FOR EACH ROW INSERT INTO t1 VALUES (1); +SET max_statement_time= default; +DROP TRIGGER IF EXISTS trg16; +Warnings: +Note 1360 Trigger does not exist +DROP TABLE t1; +# +# End of 10.3 tests +# diff --git a/mysql-test/main/lock_kill.test b/mysql-test/main/lock_kill.test new file mode 100644 index 00000000000..d0b83fe1413 --- /dev/null +++ b/mysql-test/main/lock_kill.test @@ -0,0 +1,59 @@ +--source include/have_innodb.inc + +# This test file is for testing killing of queries that are under LOCK TABLES + +--echo # +--echo # MDEV-17749 Kill during LOCK TABLE ; ALTER TABLE causes assert +--echo # + +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +--connect (con1,localhost,root,,test) +LOCK TABLE t1 WRITE; +--let $conid= `SELECT CONNECTION_ID()` +--send ALTER TABLE t1 ADD COLUMN b INT +--connection default +--disable_query_log +--echo Killing connection +eval KILL $conid; +--enable_query_log +--connection con1 +--error 0,2013 +reap; +--connection default +--disconnect con1 +DROP TABLE t1; + +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +CREATE TABLE t2 (a INT) ENGINE=InnoDB; +--connect (con1,localhost,root,,test) +LOCK TABLE t1 WRITE, t2 WRITE; +--let $conid= `SELECT CONNECTION_ID()` +--send ALTER TABLE t1 ADD COLUMN b INT +--connection default +--disable_query_log +--echo Killing connection +eval KILL $conid; +--enable_query_log +--connection con1 +--error 0,2013 +reap; +--connection default +--disconnect con1 +DROP TABLE t1, t2; + +# Similar test for CREATE TRIGGER, which also failed + +CREATE TABLE t1 (id int(11)) ENGINE=InnoDB; +LOCK TABLES t1 WRITE; +SET max_statement_time= 0.0001; +--error 0,1969 +--disable_warnings +CREATE TRIGGER tr16 AFTER UPDATE ON t1 FOR EACH ROW INSERT INTO t1 VALUES (1); +--enable_warnings +SET max_statement_time= default; +DROP TRIGGER IF EXISTS trg16; +DROP TABLE t1; + +--echo # +--echo # End of 10.3 tests +--echo # diff --git a/sql/sql_base.cc b/sql/sql_base.cc index f793556cf62..e80a78ff802 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -2469,7 +2469,17 @@ unlink_all_closed_tables(THD *thd, MYSQL_LOCK *lock, size_t reopen_count) /* If no tables left, do an automatic UNLOCK TABLES */ if (thd->lock && thd->lock->table_count == 0) + { + /* + We have to rollback any open transactions here. + This is required in the case where the server has been killed + but some transations are still open (as part of locked tables). + If we don't do this, we will get an assert in unlock_locked_tables(). + */ + ha_rollback_trans(thd, FALSE); + ha_rollback_trans(thd, TRUE); unlock_locked_tables(thd); + } } From 1e5ebf3762abdb8108620b46e76d4ebdde8472f7 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 26 May 2021 14:53:26 +0300 Subject: [PATCH 10/30] Fixed results for main.delete_use_source to make it repeatable --- mysql-test/main/delete_use_source.result | 4 ++-- mysql-test/main/delete_use_source.test | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/delete_use_source.result b/mysql-test/main/delete_use_source.result index ae2f7b93208..3e2feb46cc9 100644 --- a/mysql-test/main/delete_use_source.result +++ b/mysql-test/main/delete_use_source.result @@ -64,8 +64,8 @@ rollback; start transaction; explain delete from v1 where (select count(*) from t1 b where b.c1=v1.c1) = 500; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t1 ALL c1 NULL NULL NULL 670 Using where -2 DEPENDENT SUBQUERY b ref c1 c1 4 test.t1.c1 83 Using index +1 PRIMARY t1 ALL c1 NULL NULL NULL # Using where +2 DEPENDENT SUBQUERY b ref c1 c1 4 test.t1.c1 # Using index delete from v1 where (select count(*) from t1 b where b.c1=v1.c1) = 500 ; affected rows: 500 select count(*) from v1 where c1=0; diff --git a/mysql-test/main/delete_use_source.test b/mysql-test/main/delete_use_source.test index ed4c62b090c..4aed00da375 100644 --- a/mysql-test/main/delete_use_source.test +++ b/mysql-test/main/delete_use_source.test @@ -64,6 +64,7 @@ rollback; --echo # start transaction; +--replace_column 9 # explain delete from v1 where (select count(*) from t1 b where b.c1=v1.c1) = 500; --enable_info ONCE delete from v1 where (select count(*) from t1 b where b.c1=v1.c1) = 500 ; From dfbeddaa11193191c30cf78f7284eb1ccda6fd4c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 22 May 2021 17:47:42 +0200 Subject: [PATCH 11/30] MDEV-25726 get rid of cmake comment hack in sql_yacc.yy and replace it with equally unsightly %ifdef/%endif hack also, support %else, it's nice --- .gitignore | 1 + sql/CMakeLists.txt | 27 +++++++++++++----------- sql/gen_sql_yacc_ora_yy.cmake | 15 -------------- sql/gen_yy_files.cmake | 39 +++++++++++++++++++++++++++++++++++ sql/myskel.m4 | 18 ++++++++++++++++ sql/sql_lex.h | 2 +- sql/sql_yacc.yy | 26 +++++++++-------------- 7 files changed, 84 insertions(+), 44 deletions(-) delete mode 100644 sql/gen_sql_yacc_ora_yy.cmake create mode 100644 sql/gen_yy_files.cmake create mode 100644 sql/myskel.m4 diff --git a/.gitignore b/.gitignore index 21350997aa5..a001d521728 100644 --- a/.gitignore +++ b/.gitignore @@ -198,6 +198,7 @@ sql/mysqld sql/sql_builtin.cc sql/sql_yacc.cc sql/sql_yacc.hh +sql/sql_yacc_mariadb.yy sql/sql_yacc_ora.cc sql/sql_yacc_ora.hh sql/sql_yacc_ora.yy diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index bff01ca817e..951067a0d3d 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -66,16 +66,17 @@ ADD_CUSTOM_COMMAND( ) ADD_CUSTOM_COMMAND( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy - COMMAND ${CMAKE_COMMAND} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_mariadb.yy + ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy + COMMAND ${CMAKE_COMMAND} "-DVAL1=ORACLE" "-DVAL2=MARIADB" + "-DOUT1=${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy" + "-DOUT2=${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_mariadb.yy" "-DIN=${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy" - "-DOUT=${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy" - -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_sql_yacc_ora_yy.cmake + -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_yy_files.cmake DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy + ${CMAKE_CURRENT_SOURCE_DIR}/gen_yy_files.cmake ) -ADD_CUSTOM_TARGET(gen_sql_yacc_ora_yy DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy) - ADD_DEFINITIONS(-DMYSQL_SERVER -DHAVE_EVENT_SCHEDULER) IF(SSL_DEFINES) @@ -350,10 +351,13 @@ IF (NOT BISON_FOUND) MESSAGE(FATAL_ERROR ${ERRMSG}) ENDIF() ELSE() - BISON_TARGET(gen_sql_yacc ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc - COMPILE_FLAGS "-p MYSQL") - BISON_TARGET(gen_sql_yacc_ora ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc - COMPILE_FLAGS "-p ORA") + BISON_TARGET(gen_mariadb_cc_hh ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_mariadb.yy + ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc + DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.hh + COMPILE_FLAGS "-p MYSQL -S ${CMAKE_CURRENT_SOURCE_DIR}/myskel.m4") + BISON_TARGET(gen_oracle_cc_hh ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy + ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc + COMPILE_FLAGS "-p ORA -S ${CMAKE_CURRENT_SOURCE_DIR}/myskel.m4") ENDIF() IF(NOT CMAKE_CROSSCOMPILING OR DEFINED CMAKE_CROSSCOMPILING_EMULATOR) @@ -377,11 +381,10 @@ ADD_CUSTOM_TARGET( DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h + ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc ) -ADD_DEPENDENCIES(GenServerSource gen_sql_yacc_ora_yy) - IF(WIN32 OR HAVE_DLOPEN AND NOT DISABLE_SHARED) ADD_LIBRARY(udf_example MODULE udf_example.c udf_example.def) SET_TARGET_PROPERTIES(udf_example PROPERTIES PREFIX "") diff --git a/sql/gen_sql_yacc_ora_yy.cmake b/sql/gen_sql_yacc_ora_yy.cmake deleted file mode 100644 index 3fdd5d43f8d..00000000000 --- a/sql/gen_sql_yacc_ora_yy.cmake +++ /dev/null @@ -1,15 +0,0 @@ - -file(READ "${IN}" yytmp) - -# Comment out sql_mode=DEFAULT rules and directives (e.g. %expect, %type) -string(REPLACE "/* Start SQL_MODE_DEFAULT_SPECIFIC */" - "/* Start SQL_MODE_DEFAULT_SPECIFIC" yytmp "${yytmp}") -string(REPLACE "/* End SQL_MODE_DEFAULT_SPECIFIC */" - "End SQL_MODE_DEFAULT_SPECIFIC */" yytmp "${yytmp}") - -# Uncomment sql_mode=ORACLE rules and directives -string(REPLACE "/* Start SQL_MODE_ORACLE_SPECIFIC" - "/* Start SQL_MODE_ORACLE_SPECIFIC */" yytmp "${yytmp}") -string(REPLACE "End SQL_MODE_ORACLE_SPECIFIC */" - "/* End SQL_MODE_ORACLE_SPECIFIC */" yytmp "${yytmp}") -file(WRITE "${OUT}" "${yytmp}") diff --git a/sql/gen_yy_files.cmake b/sql/gen_yy_files.cmake new file mode 100644 index 00000000000..da63c72c37c --- /dev/null +++ b/sql/gen_yy_files.cmake @@ -0,0 +1,39 @@ +if(POLICY CMP0054) + cmake_policy(SET CMP0054 NEW) +endif() +file(READ "${IN}" data) +file(WRITE "${OUT1}" "") +file(WRITE "${OUT2}" "") +set(where 0) +string(REGEX REPLACE "/\\* sql_yacc\\.yy \\*/" "/* DON'T EDIT THIS FILE. IT'S GENERATED. EDIT sql_yacc.yy INSTEAD */" data "${data}") +while(NOT data STREQUAL "") + string(REGEX MATCH "^(%[ie][^\n]*\n)|((%[^ie\n]|[^%\n])[^\n]*\n)+|\n+" line "${data}") + string(LENGTH "${line}" ll) + string(SUBSTRING "${data}" ${ll} -1 data) + + if (line MATCHES "^%ifdef +${VAL1} *\n") + set(where 1) + set(line "\n") + elseif(line MATCHES "^%ifdef +${VAL2} *\n") + set(where 2) + set(line "\n") + elseif(line MATCHES "^%else( *| +.*)\n" AND where GREATER 0) + math(EXPR where "3-${where}") + set(line "\n") + elseif(line MATCHES "^%endif( *| +.*)\n") + set(where 0) + set(line "\n") + endif() + if(where STREQUAL 1) + file(APPEND "${OUT1}" "${line}") + string(REGEX REPLACE "[^\n]+" "" line "${line}") + file(APPEND "${OUT2}" "${line}") + elseif(where STREQUAL 2) + file(APPEND "${OUT2}" "${line}") + string(REGEX REPLACE "[^\n]+" "" line "${line}") + file(APPEND "${OUT1}" "${line}") + else() + file(APPEND "${OUT1}" "${line}") + file(APPEND "${OUT2}" "${line}") + endif() +endwhile() diff --git a/sql/myskel.m4 b/sql/myskel.m4 new file mode 100644 index 00000000000..b26fe46d342 --- /dev/null +++ b/sql/myskel.m4 @@ -0,0 +1,18 @@ +# +# fix the #line directives in the generated .cc files +# to refer to the original sql_yacc.yy +# +m4_define([yyfile],m4_bpatsubst(__file__,[[a-z.0-9]+$],sql_yacc.yy)) + +m4_define([b4_syncline], +[m4_if(m4_index([$2],[.yy]),[-1], +[b4_sync_start([$1], [$2])[]dnl + +],[b4_sync_start([$1], ["yyfile"])[]dnl + +])]) + +# try both paths for different bison versions +m4_sinclude(skeletons/c-skel.m4) +m4_sinclude(c-skel.m4) + diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 4c15c755aa7..0857c469a1c 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -355,7 +355,7 @@ void binlog_unsafe_map_init(); #ifdef MYSQL_SERVER /* - The following hack is needed because mysql_yacc.cc does not define + The following hack is needed because sql_yacc.cc does not define YYSTYPE before including this file */ #ifdef MYSQL_YACC diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index ec72e39bae3..41bfe514235 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -341,14 +341,11 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); We should not introduce any further shift/reduce conflicts. */ -/* Start SQL_MODE_DEFAULT_SPECIFIC */ +%ifdef MARIADB %expect 67 -/* End SQL_MODE_DEFAULT_SPECIFIC */ - - -/* Start SQL_MODE_ORACLE_SPECIFIC +%else %expect 69 -End SQL_MODE_ORACLE_SPECIFIC */ +%endif /* @@ -1766,7 +1763,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ %type history_point %type with_or_without_system -/* Start SQL_MODE_DEFAULT_SPECIFIC */ +%ifdef MARIADB %type sp_tail_standalone %type sp_unlabeled_block_not_atomic %type sp_proc_stmt_in_returns_clause @@ -1778,10 +1775,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ %type sp_decl_variable_list %type sp_decl_variable_list_anchored %type reserved_keyword_udt_param_type -/* End SQL_MODE_DEFAULT_SPECIFIC */ - - -/* Start SQL_MODE_ORACLE_SPECIFIC +%else %type set_assign %type sp_opt_inout %type sp_tail_standalone @@ -1816,7 +1810,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ %type package_routine_lex %type package_specification_function %type package_specification_procedure -End SQL_MODE_ORACLE_SPECIFIC */ +%endif ORACLE %% @@ -17809,7 +17803,7 @@ _empty: /* Empty */ ; -/* Start SQL_MODE_DEFAULT_SPECIFIC */ +%ifdef MARIADB statement: @@ -18208,10 +18202,10 @@ sp_unlabeled_block_not_atomic: ; -/* End SQL_MODE_DEFAULT_SPECIFIC */ +%endif MARIADB -/* Start SQL_MODE_ORACLE_SPECIFIC +%ifdef ORACLE statement: verb_clause @@ -19220,7 +19214,7 @@ sp_block_statements_and_exceptions: } ; -End SQL_MODE_ORACLE_SPECIFIC */ +%endif ORACLE /** @} (end of group Parser) From 4777097feef0016ac298f2d278348b0088e6a0b1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 23 May 2021 18:38:46 +0200 Subject: [PATCH 12/30] followup: rename generated files to have distinct names --- .gitignore | 12 ++++++------ cmake/make_dist.cmake.in | 16 ++++++++-------- libmysqld/CMakeLists.txt | 6 +++--- sql/CMakeLists.txt | 36 ++++++++++++++++++------------------ sql/gen_lex_token.cc | 2 +- sql/item_func.h | 2 +- sql/item_sum.h | 2 +- sql/sql_digest.cc | 2 +- sql/sql_lex.h | 8 ++++---- sql/sql_parse.cc | 4 ++-- sql/sql_priv.h | 2 +- sql/sql_yacc.yy | 4 ++-- 12 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index a001d521728..35b2cc4200e 100644 --- a/.gitignore +++ b/.gitignore @@ -196,12 +196,12 @@ sql/lex_hash.h sql/mysql_tzinfo_to_sql sql/mysqld sql/sql_builtin.cc -sql/sql_yacc.cc -sql/sql_yacc.hh -sql/sql_yacc_mariadb.yy -sql/sql_yacc_ora.cc -sql/sql_yacc_ora.hh -sql/sql_yacc_ora.yy +sql/yy_mariadb.cc +sql/yy_mariadb.hh +sql/yy_mariadb.yy +sql/yy_oracle.cc +sql/yy_oracle.hh +sql/yy_oracle.yy storage/heap/hp_test1 storage/heap/hp_test2 storage/maria/aria_chk diff --git a/cmake/make_dist.cmake.in b/cmake/make_dist.cmake.in index 9e9eb555951..4f6988cfed2 100644 --- a/cmake/make_dist.cmake.in +++ b/cmake/make_dist.cmake.in @@ -125,15 +125,15 @@ IF(NOT GIT_EXECUTABLE) ENDIF() # Copy bison output -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.hh - ${PACKAGE_DIR}/sql/sql_yacc.hh COPYONLY) -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.cc - ${PACKAGE_DIR}/sql/sql_yacc.cc COPYONLY) +CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/yy_mariadb.hh + ${PACKAGE_DIR}/sql/yy_mariadb.hh COPYONLY) +CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/yy_mariadb.cc + ${PACKAGE_DIR}/sql/yy_mariadb.cc COPYONLY) # Copy bison output -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.hh - ${PACKAGE_DIR}/sql/sql_yacc_ora.hh COPYONLY) -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.cc - ${PACKAGE_DIR}/sql/sql_yacc_ora.cc COPYONLY) +CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/yy_oracle.hh + ${PACKAGE_DIR}/sql/yy_oracle.hh COPYONLY) +CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/yy_oracle.cc + ${PACKAGE_DIR}/sql/yy_oracle.cc COPYONLY) # Add documentation, if user has specified where to find them IF(MYSQL_DOCS_LOCATION) diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 90b27a59f02..c133e5a4378 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -31,9 +31,9 @@ ${SSL_INTERNAL_INCLUDE_DIRS} SET(GEN_SOURCES ${CMAKE_BINARY_DIR}/sql/sql_yacc.hh -${CMAKE_BINARY_DIR}/sql/sql_yacc.cc -${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.hh -${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.cc +${CMAKE_BINARY_DIR}/sql/yy_mariadb.cc +${CMAKE_BINARY_DIR}/sql/yy_oracle.hh +${CMAKE_BINARY_DIR}/sql/yy_oracle.cc ${CMAKE_BINARY_DIR}/sql/lex_hash.h ) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 951067a0d3d..eb6f3d4dfdd 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -66,13 +66,14 @@ ADD_CUSTOM_COMMAND( ) ADD_CUSTOM_COMMAND( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_mariadb.yy - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.yy + ${CMAKE_CURRENT_BINARY_DIR}/yy_oracle.yy COMMAND ${CMAKE_COMMAND} "-DVAL1=ORACLE" "-DVAL2=MARIADB" - "-DOUT1=${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy" - "-DOUT2=${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_mariadb.yy" + "-DOUT1=${CMAKE_CURRENT_BINARY_DIR}/yy_oracle.yy" + "-DOUT2=${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.yy" "-DIN=${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy" -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_yy_files.cmake + COMMENT "Building yy_mariadb.yy and yy_oracle.yy from sql_yacc.yy" DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy ${CMAKE_CURRENT_SOURCE_DIR}/gen_yy_files.cmake ) @@ -84,8 +85,8 @@ IF(SSL_DEFINES) ENDIF() SET (SQL_SOURCE - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc + ${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.cc + ${CMAKE_CURRENT_BINARY_DIR}/yy_oracle.cc ../sql-common/client.c compat56.cc derror.cc des_key_file.cc discover.cc ../sql-common/errmsg.c field.cc field_conv.cc field_comp.cc @@ -328,7 +329,7 @@ FIND_PACKAGE(BISON 2.0) # there IF (NOT BISON_FOUND) IF (NOT ${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR}) - FOREACH(file sql_yacc.cc sql_yacc.hh sql_yacc_ora.cc sql_yacc_ora.hh) + FOREACH(file yy_mariadb.cc yy_mariadb.hh yy_oracle.cc yy_oracle.hh) IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${file} AND (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${file})) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${file} ${CMAKE_CURRENT_BINARY_DIR}/${file} COPYONLY) @@ -336,7 +337,7 @@ IF (NOT BISON_FOUND) ENDFOREACH() ENDIF() - IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc) + IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.cc) # Output files are missing, bail out. SET(ERRMSG "Bison (GNU parser generator) is required to build MySQL." @@ -351,18 +352,17 @@ IF (NOT BISON_FOUND) MESSAGE(FATAL_ERROR ${ERRMSG}) ENDIF() ELSE() - BISON_TARGET(gen_mariadb_cc_hh ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_mariadb.yy - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc - DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.hh + BISON_TARGET(gen_mariadb_cc_hh ${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.yy + ${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.cc COMPILE_FLAGS "-p MYSQL -S ${CMAKE_CURRENT_SOURCE_DIR}/myskel.m4") - BISON_TARGET(gen_oracle_cc_hh ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.yy - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc + BISON_TARGET(gen_oracle_cc_hh ${CMAKE_CURRENT_BINARY_DIR}/yy_oracle.yy + ${CMAKE_CURRENT_BINARY_DIR}/yy_oracle.cc COMPILE_FLAGS "-p ORA -S ${CMAKE_CURRENT_SOURCE_DIR}/myskel.m4") ENDIF() IF(NOT CMAKE_CROSSCOMPILING OR DEFINED CMAKE_CROSSCOMPILING_EMULATOR) ADD_EXECUTABLE(gen_lex_token gen_lex_token.cc - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.hh) + ${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.hh) ADD_EXECUTABLE(gen_lex_hash gen_lex_hash.cc) ENDIF() @@ -381,8 +381,8 @@ ADD_CUSTOM_TARGET( DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc + ${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.cc + ${CMAKE_CURRENT_BINARY_DIR}/yy_oracle.cc ) IF(WIN32 OR HAVE_DLOPEN AND NOT DISABLE_SHARED) @@ -397,8 +397,8 @@ CONFIGURE_FILE( ADD_CUSTOM_TARGET(dist COMMAND ${CMAKE_COMMAND} -P ${CMAKE_BINARY_DIR}/make_dist.cmake - DEPENDS ${CMAKE_BINARY_DIR}/sql/sql_yacc.cc ${CMAKE_BINARY_DIR}/sql/sql_yacc.hh - DEPENDS ${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.cc ${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.hh + DEPENDS ${CMAKE_BINARY_DIR}/sql/yy_mariadb.cc ${CMAKE_BINARY_DIR}/sql/yy_mariadb.hh + DEPENDS ${CMAKE_BINARY_DIR}/sql/yy_oracle.cc ${CMAKE_BINARY_DIR}/sql/yy_oracle.hh WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) diff --git a/sql/gen_lex_token.cc b/sql/gen_lex_token.cc index 0ca03b0bf7b..40145459917 100644 --- a/sql/gen_lex_token.cc +++ b/sql/gen_lex_token.cc @@ -19,7 +19,7 @@ /* We only need the tokens here */ #define YYSTYPE_IS_DECLARED -#include +#include #include #include /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ diff --git a/sql/item_func.h b/sql/item_func.h index e774d9c53bd..a7dbb9ab4bc 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -2761,7 +2761,7 @@ public: { return get_item_copy(thd, this); } }; -#else /* Dummy functions to get sql_yacc.cc compiled */ +#else /* Dummy functions to get yy_*.cc files compiled */ class Item_func_udf_float :public Item_real_func { diff --git a/sql/item_sum.h b/sql/item_sum.h index 118f78ec5c1..b8f38ad96bc 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -1766,7 +1766,7 @@ public: { return get_item_copy(thd, this); } }; -#else /* Dummy functions to get sql_yacc.cc compiled */ +#else /* Dummy functions to get yy_*.cc files compiled */ class Item_sum_udf_float :public Item_sum_double { diff --git a/sql/sql_digest.cc b/sql/sql_digest.cc index 5ca855c9608..b81a54b2af6 100644 --- a/sql/sql_digest.cc +++ b/sql/sql_digest.cc @@ -32,7 +32,7 @@ #include "sql_get_diagnostics.h" /* Generated code */ -#include "sql_yacc.hh" +#include "yy_mariadb.hh" #define LEX_TOKEN_WITH_DEFINITION #include "lex_token.h" diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 0857c469a1c..d0ccf19ddab 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -355,7 +355,7 @@ void binlog_unsafe_map_init(); #ifdef MYSQL_SERVER /* - The following hack is needed because sql_yacc.cc does not define + The following hack is needed because yy_*.cc do not define YYSTYPE before including this file */ #ifdef MYSQL_YACC @@ -363,10 +363,10 @@ void binlog_unsafe_map_init(); #else #include "lex_symbol.h" #ifdef MYSQL_LEX -#include "item_func.h" /* Cast_target used in sql_yacc.hh */ -#include "sql_get_diagnostics.h" /* Types used in sql_yacc.hh */ +#include "item_func.h" /* Cast_target used in yy_mariadb.hh */ +#include "sql_get_diagnostics.h" /* Types used in yy_mariadb.hh */ #include "sp_pcontext.h" -#include "sql_yacc.hh" +#include "yy_mariadb.hh" #define LEX_YYSTYPE YYSTYPE * #else #define LEX_YYSTYPE void * diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 68344b49857..63898ad4db7 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -10374,8 +10374,8 @@ bool check_host_name(LEX_CSTRING *str) } -extern int MYSQLparse(THD *thd); // from sql_yacc.cc -extern int ORAparse(THD *thd); // from sql_yacc_ora.cc +extern int MYSQLparse(THD *thd); // from yy_mariadb.cc +extern int ORAparse(THD *thd); // from yy_oracle.cc /** diff --git a/sql/sql_priv.h b/sql/sql_priv.h index 07f07a7150f..2206f71c060 100644 --- a/sql/sql_priv.h +++ b/sql/sql_priv.h @@ -392,7 +392,7 @@ enum enum_yes_no_unknown */ -/* sql_yacc.cc */ +/* yy_*.cc */ #ifndef DBUG_OFF extern void turn_parser_debug_on_MYSQLparse(); extern void turn_parser_debug_on_ORAparse(); diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 41bfe514235..c9bc2bdc6a6 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -165,7 +165,7 @@ static void yyerror(THD *thd, const char *s) void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() { /* - MYSQLdebug is in sql/sql_yacc.cc, in bison generated code. + MYSQLdebug is in sql/yy_*.cc, in bison generated code. Turning this option on is **VERY** verbose, and should be used when investigating a syntax error problem only. @@ -17791,7 +17791,7 @@ uninstall: } ; -/* Avoid compiler warning from sql_yacc.cc where yyerrlab1 is not used */ +/* Avoid compiler warning from yy_*.cc where yyerrlab1 is not used */ keep_gcc_happy: IMPOSSIBLE_ACTION { From ab87fc6c7ad4ce261a833330e939facc4358612a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 27 May 2021 09:31:19 +0300 Subject: [PATCH 13/30] Cleanup: Remove handler::update_table_comment() The only call of the virtual member function handler::update_table_comment() was removed in commit 82d28fada7dc928564aefac802400c6684c11917 (MySQL 5.5.53) but the implementation was not removed. The only non-trivial implementation was for InnoDB. The information is now returned via handler::get_foreign_key_create_info() and ha_statistics::delete_length. --- sql/ha_partition.cc | 21 +-------- sql/ha_partition.h | 7 +-- sql/handler.h | 4 +- storage/innobase/handler/ha_innodb.cc | 62 --------------------------- storage/innobase/handler/ha_innodb.h | 1 - 5 files changed, 3 insertions(+), 92 deletions(-) diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 523e76e511e..ad020acf5c4 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2005, 2019, Oracle and/or its affiliates. - Copyright (c) 2009, 2019, MariaDB + Copyright (c) 2009, 2021, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2244,25 +2244,6 @@ void ha_partition::change_table_ptr(TABLE *table_arg, TABLE_SHARE *share) } } -/* - Change comments specific to handler - - SYNOPSIS - update_table_comment() - comment Original comment - - RETURN VALUE - new comment - - DESCRIPTION - No comment changes so far -*/ - -char *ha_partition::update_table_comment(const char *comment) -{ - return (char*) comment; /* Nothing to change */ -} - /** Handle delete and rename table diff --git a/sql/ha_partition.h b/sql/ha_partition.h index a48aa639237..34174089bf6 100644 --- a/sql/ha_partition.h +++ b/sql/ha_partition.h @@ -3,7 +3,7 @@ /* Copyright (c) 2005, 2012, Oracle and/or its affiliates. - Copyright (c) 2009, 2013, Monty Program Ab & SkySQL Ab. + Copyright (c) 2009, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -314,10 +314,6 @@ public: Meta data routines to CREATE, DROP, RENAME table and often used at ALTER TABLE (update_create_info used from ALTER TABLE and SHOW ..). - update_table_comment is used in SHOW TABLE commands to provide a - chance for the handler to add any interesting comments to the table - comments not provided by the users comment. - create_partitioning_metadata is called before opening a new handler object with openfrm to call create. It is used to create any local handler object needed in opening the object in openfrm @@ -330,7 +326,6 @@ public: virtual int create_partitioning_metadata(const char *name, const char *old_name, int action_flag); virtual void update_create_info(HA_CREATE_INFO *create_info); - virtual char *update_table_comment(const char *comment); virtual int change_partitions(HA_CREATE_INFO *create_info, const char *path, ulonglong * const copied, diff --git a/sql/handler.h b/sql/handler.h index f618c1d6469..b3617d7a47a 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -2,7 +2,7 @@ #define HANDLER_INCLUDED /* Copyright (c) 2000, 2019, Oracle and/or its affiliates. - Copyright (c) 2009, 2019, MariaDB + Copyright (c) 2009, 2021, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -3363,8 +3363,6 @@ public: /* end of the list of admin commands */ virtual int indexes_are_disabled(void) {return 0;} - virtual char *update_table_comment(const char * comment) - { return (char*) comment;} virtual void append_create_info(String *packet) {} /** If index == MAX_KEY then a check for table is made and if index < diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5617a293a23..81f1dae85b6 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -15190,68 +15190,6 @@ ha_innobase::check( DBUG_RETURN(is_ok ? HA_ADMIN_OK : HA_ADMIN_CORRUPT); } -/*************************************************************//** -Adds information about free space in the InnoDB tablespace to a table comment -which is printed out when a user calls SHOW TABLE STATUS. Adds also info on -foreign keys. -@return table comment + InnoDB free space + info on foreign keys */ -UNIV_INTERN -char* -ha_innobase::update_table_comment( -/*==============================*/ - const char* comment)/*!< in: table comment defined by user */ -{ - uint length = (uint) strlen(comment); - char* str=0; - size_t flen; - std::string fk_str; - - /* We do not know if MySQL can call this function before calling - external_lock(). To be safe, update the thd of the current table - handle. */ - - if (length > 64000 - 3) { - return((char*) comment); /* string too long */ - } - - update_thd(ha_thd()); - - m_prebuilt->trx->op_info = "returning table comment"; - -#define SSTR( x ) reinterpret_cast< std::ostringstream & >( \ - ( std::ostringstream() << std::dec << x ) ).str() - - fk_str.append("InnoDB free: "); - fk_str.append(SSTR(fsp_get_available_space_in_free_extents( - m_prebuilt->table->space))); - - fk_str.append(dict_print_info_on_foreign_keys( - FALSE, m_prebuilt->trx, - m_prebuilt->table)); - - flen = fk_str.length(); - - if (length + flen + 3 > 64000) { - flen = 64000 - 3 - length; - } - /* allocate buffer for the full string */ - str = (char*) my_malloc(length + flen + 3, MYF(0)); - if (str) { - char* pos = str + length; - if (length) { - memcpy(str, comment, length); - *pos++ = ';'; - *pos++ = ' '; - } - memcpy(pos, fk_str.c_str(), flen); - pos[flen] = 0; - } - - m_prebuilt->trx->op_info = (char*)""; - - return(str ? str : (char*) comment); -} - /*******************************************************************//** Gets the foreign key create info for a table stored in InnoDB. @return own: character string in the form which can be inserted to the diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 274c6761d4f..eab009f7ac2 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -221,7 +221,6 @@ public: int rename_table(const char* from, const char* to); inline int defragment_table(const char* name); int check(THD* thd, HA_CHECK_OPT* check_opt); - char* update_table_comment(const char* comment); char* get_foreign_key_create_info(); From c11c5f36d895cc9457a0945aa7bd043168049fef Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 24 May 2021 19:40:47 +0530 Subject: [PATCH 14/30] MDEV-25758 InnoDB spatial indexes miss large geometry fields after MDEV-25459 InnoDB should calculate the MBR for the first field of spatial index and do the comparison with the clustered index field MBR. Due to MDEV-25459 refactoring, InnoDB calculate the length of the first field and fails with too long column error. --- mysql-test/suite/innodb_gis/r/gis.result | 13 +- mysql-test/suite/innodb_gis/t/gis.test | 15 ++- storage/innobase/row/row0sel.cc | 147 +++++++++++++++-------- 3 files changed, 119 insertions(+), 56 deletions(-) diff --git a/mysql-test/suite/innodb_gis/r/gis.result b/mysql-test/suite/innodb_gis/r/gis.result index 33e0c1c0410..3a1a1aa28fd 100644 --- a/mysql-test/suite/innodb_gis/r/gis.result +++ b/mysql-test/suite/innodb_gis/r/gis.result @@ -1482,10 +1482,21 @@ FROM buildings, bridges WHERE ST_Contains(ST_Buffer(bridges.position, 15.0), buildings.footprint) = 1; count(*) 1 -DROP DATABASE gis_ogs; # # Bug#13362660 ASSERTION `FIELD_POS < FIELD_COUNT' FAILED. IN PROTOCOL_TEXT::STORE # SELECT ST_Union('', ''), md5(1); ST_Union('', '') md5(1) NULL c4ca4238a0b923820dcc509a6f75849b +# +# MDEV-25758 InnoDB spatial indexes miss large geometry +# fields after MDEV-25459 +# +CREATE TABLE t1(l LINESTRING NOT NULL, SPATIAL INDEX(l))ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +SELECT GROUP_CONCAT(CONCAT(seq, ' ', seq) SEPARATOR ',') INTO @g FROM seq_0_to_190; +INSERT INTO t1 SET l=ST_GeomFromText(CONCAT('LINESTRING(',@g,',0 0)')); +SELECT COUNT(*) FROM t1 WHERE MBRIntersects(GeomFromText('Polygon((0 0,0 10,10 10,10 0,0 0))'), l); +COUNT(*) +1 +DROP TABLE t1; +DROP DATABASE gis_ogs; diff --git a/mysql-test/suite/innodb_gis/t/gis.test b/mysql-test/suite/innodb_gis/t/gis.test index ba1b3b12ec2..acc2b1e36d8 100644 --- a/mysql-test/suite/innodb_gis/t/gis.test +++ b/mysql-test/suite/innodb_gis/t/gis.test @@ -2,6 +2,7 @@ --source include/have_innodb.inc -- source include/have_geometry.inc +--source include/have_sequence.inc SET default_storage_engine=InnoDB; @@ -1422,10 +1423,20 @@ WHERE ST_Contains(ST_Buffer(bridges.position, 15.0), buildings.footprint) = 1; #FROM lakes #WHERE lakes.name = 'Blue Lake'; -DROP DATABASE gis_ogs; - --echo # --echo # Bug#13362660 ASSERTION `FIELD_POS < FIELD_COUNT' FAILED. IN PROTOCOL_TEXT::STORE --echo # SELECT ST_Union('', ''), md5(1); + +--echo # +--echo # MDEV-25758 InnoDB spatial indexes miss large geometry +--echo # fields after MDEV-25459 +--echo # +CREATE TABLE t1(l LINESTRING NOT NULL, SPATIAL INDEX(l))ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +SELECT GROUP_CONCAT(CONCAT(seq, ' ', seq) SEPARATOR ',') INTO @g FROM seq_0_to_190; +INSERT INTO t1 SET l=ST_GeomFromText(CONCAT('LINESTRING(',@g,',0 0)')); +SELECT COUNT(*) FROM t1 WHERE MBRIntersects(GeomFromText('Polygon((0 0,0 10,10 10,10 0,0 0))'), l); +DROP TABLE t1; + +DROP DATABASE gis_ogs; diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 86cb6b0ea1c..83bf18ab9b8 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -150,6 +150,75 @@ row_sel_sec_rec_is_for_blob( return(!cmp_data_data(mtype, prtype, buf, len, sec_field, sec_len)); } +/** Function to read the secondary spatial index, calculate +the minimum bounding rectangle for clustered index record +and secondary index record and compare it. +@param sec_rec secondary index record +@param sec_index spatial secondary index +@param clust_rec clustered index record +@param clust_index clustered index +@retval DB_SUCCESS_LOCKED_REC if the secondary record is equal to the + corresponding fields in the clustered record, when compared with + collation; +@retval DB_SUCCESS if not equal */ +static +dberr_t +row_sel_spatial_sec_rec_is_for_clust_rec( + const rec_t *sec_rec, const dict_index_t *sec_index, + const rec_t *clust_rec, dict_index_t *clust_index) +{ + mem_heap_t *heap= mem_heap_create(256); + rec_offs clust_offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs *clust_offs= clust_offsets_; + ulint clust_len; + + rec_offs_init(clust_offsets_); + ulint clust_pos= dict_col_get_clust_pos( + dict_index_get_nth_col(sec_index, 0), clust_index); + clust_offs= rec_get_offsets(clust_rec, clust_index, clust_offs, + true, clust_pos + 1, &heap); + ut_ad(sec_index->n_user_defined_cols == 1); + const byte *clust_field= rec_get_nth_field(clust_rec, clust_offs, + clust_pos, &clust_len); + if (clust_len == UNIV_SQL_NULL || clust_len < GEO_DATA_HEADER_SIZE) + { + ut_ad("corrupted geometry column" == 0); +err_exit: + mem_heap_free(heap); + return DB_SUCCESS; + } + + /* For externally stored field, we need to get full + geo data to generate the MBR for comparing. */ + if (rec_offs_nth_extern(clust_offs, clust_pos)) + { + clust_field= btr_copy_externally_stored_field( + &clust_len, clust_field, dict_table_page_size(sec_index->table), + clust_len, heap); + if (clust_field == NULL) + { + ut_ad("corrupted geometry blob" == 0); + goto err_exit; + } + } + + ut_ad(clust_len >= GEO_DATA_HEADER_SIZE); + rtr_mbr_t tmp_mbr; + rtr_mbr_t sec_mbr; + + rtree_mbr_from_wkb( + clust_field + GEO_DATA_HEADER_SIZE, + static_cast(clust_len - GEO_DATA_HEADER_SIZE), + SPDIMS, reinterpret_cast(&tmp_mbr)); + + rtr_read_mbr(sec_rec, &sec_mbr); + + mem_heap_free(heap); + return MBR_EQUAL_CMP(&sec_mbr, &tmp_mbr) + ? DB_SUCCESS_LOCKED_REC + : DB_SUCCESS; +} + /** Returns TRUE if the user-defined column values in a secondary index record are alphabetically the same as the corresponding columns in the clustered index record. @@ -177,20 +246,6 @@ row_sel_sec_rec_is_for_clust_rec( dict_index_t* clust_index, que_thr_t* thr) { - const byte* sec_field; - ulint sec_len; - const byte* clust_field; - ulint n; - ulint i; - mem_heap_t* heap = NULL; - rec_offs clust_offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs sec_offsets_[REC_OFFS_SMALL_SIZE]; - rec_offs* clust_offs = clust_offsets_; - rec_offs* sec_offs = sec_offsets_; - - rec_offs_init(clust_offsets_); - rec_offs_init(sec_offsets_); - if (rec_get_deleted_flag(clust_rec, dict_table_is_comp(clust_index->table))) { /* In delete-marked records, DB_TRX_ID must @@ -204,7 +259,27 @@ row_sel_sec_rec_is_for_clust_rec( return DB_SUCCESS; } - heap = mem_heap_create(256); + if (sec_index->is_spatial()) { + return row_sel_spatial_sec_rec_is_for_clust_rec( + sec_rec, sec_index, clust_rec, + clust_index); + } + + const byte* sec_field; + ulint sec_len; + const byte* clust_field; + ulint n; + ulint i; + mem_heap_t* heap = mem_heap_create(256); + rec_offs clust_offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs sec_offsets_[REC_OFFS_SMALL_SIZE]; + rec_offs* clust_offs = clust_offsets_; + rec_offs* sec_offs = sec_offsets_; + + rec_offs_init(clust_offsets_); + rec_offs_init(sec_offsets_); + + ib_vcol_row vc(heap); clust_offs = rec_get_offsets(clust_rec, clust_index, clust_offs, @@ -310,44 +385,10 @@ check_for_blob: } } - /* For spatial index, the first field is MBR, we check - if the MBR is equal or not. */ - if (dict_index_is_spatial(sec_index) && i == 0) { - rtr_mbr_t tmp_mbr; - rtr_mbr_t sec_mbr; - byte* dptr = - const_cast(clust_field); - - ut_ad(clust_len != UNIV_SQL_NULL); - - /* For externally stored field, we need to get full - geo data to generate the MBR for comparing. */ - if (rec_offs_nth_extern(clust_offs, clust_pos)) { - dptr = btr_copy_externally_stored_field( - &clust_len, dptr, - dict_tf_get_page_size( - sec_index->table->flags), - len, heap); - } - - rtree_mbr_from_wkb(dptr + GEO_DATA_HEADER_SIZE, - static_cast(clust_len - - GEO_DATA_HEADER_SIZE), - SPDIMS, - reinterpret_cast( - &tmp_mbr)); - rtr_read_mbr(sec_field, &sec_mbr); - - if (!MBR_EQUAL_CMP(&sec_mbr, &tmp_mbr)) { - return DB_SUCCESS; - } - } else { - - if (0 != cmp_data_data(col->mtype, col->prtype, - clust_field, len, - sec_field, sec_len)) { - return DB_SUCCESS; - } + if (0 != cmp_data_data(col->mtype, col->prtype, + clust_field, len, + sec_field, sec_len)) { + return DB_SUCCESS; } } From ef0d883903166b7b8eaa51730662f2396ed4d88f Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Wed, 26 May 2021 15:27:07 +0300 Subject: [PATCH 15/30] Revert "Add Pull Request template file to the MariaDB/server repository" This reverts commit 17106c984b9649641068ddf108dcffd2c7773212. --- .github/pull_request_template.md | 50 -------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 6fe365495cc..00000000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,50 +0,0 @@ - - - -- [x] *The Jira issue number for this PR is: MDEV-_____* - - -## Description -TODO: fill description here - -## How can this PR be tested? -TODO: fill steps to reproduce here, if applicable, - or remove the section - - -## Basing the PR against the correct MariaDB version -- [ ] *This is a new feature and the PR is based against the latest MariaDB development branch* -- [ ] *This is a bug fix and the PR is based against the earliest branch in which the bug can be reproduced* - - -## Backward compatibility -TODO: fill details here, if applicable, or remove the section - From 08bc7ee0688b6ba3eb99d1fe4bd35e7bc97b19ca Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 27 May 2021 00:37:51 +0200 Subject: [PATCH 16/30] MDEV-25792 server hangs on early shutdown if InnoDB needs to purge indexed virtual columns if the server isn't started and innodb initiated a shutdown process - don't wait for the server to start, it won't --- .../innodb/r/index_vcol_purge_startup.result | 24 ++++++++++++++++ .../innodb/t/index_vcol_purge_startup.test | 28 +++++++++++++++++++ storage/innobase/row/row0purge.cc | 2 +- 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/innodb/r/index_vcol_purge_startup.result create mode 100644 mysql-test/suite/innodb/t/index_vcol_purge_startup.test diff --git a/mysql-test/suite/innodb/r/index_vcol_purge_startup.result b/mysql-test/suite/innodb/r/index_vcol_purge_startup.result new file mode 100644 index 00000000000..5603c14acf3 --- /dev/null +++ b/mysql-test/suite/innodb/r/index_vcol_purge_startup.result @@ -0,0 +1,24 @@ +call mtr.add_suppression('SQL_ERROR_LOG'); +call mtr.add_suppression('Failed to initialize plugins'); +call mtr.add_suppression('Aborting'); +create table t1(a int primary key, b int as (100-a*a), index(b)) engine=innodb; +insert t1 (a) values(1),(2),(3); +start transaction; +select * from t1 order by a; +a b +1 99 +2 96 +3 91 +connect con1, localhost, root; +delete from t1 where a=2; +flush tables; +connection default; +# Kill the server +# Failed restart +# Start the server +# restart +select * from t1 order by a; +a b +1 99 +3 91 +drop table t1; diff --git a/mysql-test/suite/innodb/t/index_vcol_purge_startup.test b/mysql-test/suite/innodb/t/index_vcol_purge_startup.test new file mode 100644 index 00000000000..ccc07fe7314 --- /dev/null +++ b/mysql-test/suite/innodb/t/index_vcol_purge_startup.test @@ -0,0 +1,28 @@ +source include/have_innodb.inc; +# need to restart server +source include/not_embedded.inc; + +call mtr.add_suppression('SQL_ERROR_LOG'); +call mtr.add_suppression('Failed to initialize plugins'); +call mtr.add_suppression('Aborting'); + +if (!$SQL_ERRLOG_SO) { + skip No SQL_ERROR_LOG plugin; +} + +create table t1(a int primary key, b int as (100-a*a), index(b)) engine=innodb; +insert t1 (a) values(1),(2),(3); +start transaction; +select * from t1 order by a; +connect(con1, localhost, root); +delete from t1 where a=2; +flush tables; +connection default; +source include/kill_mysqld.inc; +echo # Failed restart; +error 1; +exec $MYSQLD_LAST_CMD --plugin-load-add=$SQL_ERRLOG_SO --sql-error-log=FORCE --sql-error-log-filename=$MYSQLTEST_VARDIR/nonexistent/nonexistent; +echo # Start the server; +source include/start_mysqld.inc; +select * from t1 order by a; +drop table t1; diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 4a239bffc9e..a0600801f5a 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -957,7 +957,7 @@ already_locked: if (!mysqld_server_started) { node->close_table(); - if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) { + if (srv_shutdown_state > SRV_SHUTDOWN_NONE) { return(false); } os_thread_sleep(1000000); From 1638241e313848e9999c529ac827d18afce3e2f0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 27 May 2021 12:06:21 +0200 Subject: [PATCH 17/30] mtr: fix the debug printout to print all arguments of _verbose(), not just the number of them --- mysql-test/lib/My/SafeProcess.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/lib/My/SafeProcess.pm b/mysql-test/lib/My/SafeProcess.pm index 87054019759..4f7464daf44 100644 --- a/mysql-test/lib/My/SafeProcess.pm +++ b/mysql-test/lib/My/SafeProcess.pm @@ -628,7 +628,7 @@ sub self2str { sub _verbose { return unless $_verbose; - print STDERR " ## ". @_. "\n"; + print STDERR " ## @_\n"; } From d06205ba3713da6c5875f124d5e431d3704aad1d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 27 May 2021 14:00:58 +0200 Subject: [PATCH 18/30] CONNECT: use my_snprintf --- storage/connect/tabrest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index 4b6bb6a9e62..6679ccd55b0 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -60,12 +60,12 @@ int Xcurl(PGLOBAL g, PCSZ Http, PCSZ Uri, PCSZ filename) if (Uri) { if (*Uri == '/' || Http[strlen(Http) - 1] == '/') - sprintf(buf, "%s%s", Http, Uri); + my_snprintf(buf, sizeof(buf)-1, "%s%s", Http, Uri); else - sprintf(buf, "%s/%s", Http, Uri); + my_snprintf(buf, sizeof(buf)-1, "%s/%s", Http, Uri); } else - strcpy(buf, Http); + my_snprintf(buf, sizeof(buf)-1, "%s", Http); #if defined(__WIN__) char cmd[1024]; From a70a5537e7a048d2f555daee34e95ad3bda194ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Sun, 23 May 2021 21:23:18 -0700 Subject: [PATCH 19/30] Deb: Innotop: Add support for MariaDB 10.5+ Synced from downstream Debian: https://salsa.debian.org/mariadb-team/mariadb-10.5/-/commit/7015e8e4b5ed3a7727a8e442039fceb2c5b1a6b9 --- debian/additions/innotop/innotop | 8 ++++---- debian/additions/innotop/innotop.1 | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debian/additions/innotop/innotop b/debian/additions/innotop/innotop index 19229a57a82..23f04fe017c 100644 --- a/debian/additions/innotop/innotop +++ b/debian/additions/innotop/innotop @@ -466,7 +466,7 @@ sub parse_status_text { # too many locks to print, the output might be truncated) my $time_text; - if ( ($mysqlversion =~ /^5\.[67]\./) || ($mysqlversion =~ /^10\.[012]\./) ) { + if ( ($mysqlversion =~ /^5\.[67]\./) || ($mysqlversion =~ /^10\.[0-9]\./) ) { ( $time_text ) = $fulltext =~ m/^([0-9-]* [0-9:]*) [0-9a-fx]* INNODB MONITOR OUTPUT/m; $innodb_data{'ts'} = [ parse_innodb_timestamp_56( $time_text ) ]; } else { @@ -634,7 +634,7 @@ sub parse_fk_section { return 0 unless $fulltext; my ( $ts, $type ); - if ( ($mysqlversion =~ /^5.[67]\./) || ($mysqlversion =~ /^10.[012]\./) ) { + if ( ($mysqlversion =~ /^5.[67]\./) || ($mysqlversion =~ /^10.[0-9]\./) ) { ( $ts, $type ) = $fulltext =~ m/^([0-9-]* [0-9:]*)\s[0-9a-fx]*\s+(\w+)/m; $section->{'ts'} = [ parse_innodb_timestamp_56( $ts ) ]; } else { @@ -894,7 +894,7 @@ sub parse_dl_section { my ( $ts ) = $fulltext =~ m/^$s$/m; return 0 unless $ts; - if ( ($mysqlversion =~ /^5\.[67]\./) || ($mysqlversion =~ /^10\.[012]\./) ) { + if ( ($mysqlversion =~ /^5\.[67]\./) || ($mysqlversion =~ /^10\.[0-9]\./) ) { $dl->{'ts'} = [ parse_innodb_timestamp_56( $ts ) ]; } else { @@ -12064,7 +12064,7 @@ This data is from the TRANSACTIONS section of SHOW INNODB STATUS. =item IO_THREADS -This data is from the list of threads in the the FILE I/O section of SHOW INNODB +This data is from the list of threads in the FILE I/O section of SHOW INNODB STATUS. =item INNODB_LOCKS diff --git a/debian/additions/innotop/innotop.1 b/debian/additions/innotop/innotop.1 index 65da5dc0dfb..62a9aed69ee 100644 --- a/debian/additions/innotop/innotop.1 +++ b/debian/additions/innotop/innotop.1 @@ -2048,7 +2048,7 @@ the processlist. This data is from the \s-1TRANSACTIONS\s0 section of \s-1SHOW INNODB STATUS.\s0 .IP "\s-1IO_THREADS\s0" 4 .IX Item "IO_THREADS" -This data is from the list of threads in the the \s-1FILE I/O\s0 section of \s-1SHOW INNODB +This data is from the list of threads in the \s-1FILE I/O\s0 section of \s-1SHOW INNODB STATUS.\s0 .IP "\s-1INNODB_LOCKS\s0" 4 .IX Item "INNODB_LOCKS" From 5bd517259f2bb3e48cd3dab5fa64ce2cf9b623df Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 29 May 2021 06:19:46 +0200 Subject: [PATCH 20/30] MDEV-25815 mariabackup crash or debug assert with --backup --databases-exclude Fix regression (debug assertion or division by 0) caused by cfd3d70ccbbfcf3fdec034be446317741dfae824 --- extra/mariabackup/xtrabackup.cc | 2 +- mysql-test/suite/mariabackup/xbstream.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index a663111208e..a82d371d40d 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4048,7 +4048,7 @@ xb_register_filter_entry( strncpy(dbname, name, p - name); dbname[p - name] = 0; - if (databases_hash) { + if (databases_hash && databases_hash->array) { HASH_SEARCH(name_hash, databases_hash, ut_fold_string(dbname), xb_filter_entry_t*, diff --git a/mysql-test/suite/mariabackup/xbstream.test b/mysql-test/suite/mariabackup/xbstream.test index 9161d227b20..212ac598064 100644 --- a/mysql-test/suite/mariabackup/xbstream.test +++ b/mysql-test/suite/mariabackup/xbstream.test @@ -8,7 +8,7 @@ mkdir $targetdir; let $streamfile=$MYSQLTEST_VARDIR/tmp/backup.xb; echo # xtrabackup backup to stream; -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --databases-exclude=foobar --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; echo # xbstream extract; --disable_result_log exec $XBSTREAM -x -C $targetdir < $streamfile; From 91bde0fb67e9da6cb9ee1464ad684c0f681e47c9 Mon Sep 17 00:00:00 2001 From: Dmitry Shulga Date: Sun, 30 May 2021 17:31:55 +0700 Subject: [PATCH 21/30] MDEV-25576: The statement EXPLAIN running as regular statement and as prepared statement produces different results for UPDATE with subquery Both EXPLAIN and EXPLAIN EXTENDED statements produce different results set in case it is run in normal way and in PS mode for the statements UPDATE/DELETE with subquery. The use case below reproduces the issue: MariaDB [test]> CREATE TABLE t1 (c1 INT KEY) ENGINE=MyISAM; Query OK, 0 rows affected (0,128 sec) MariaDB [test]> CREATE TABLE t2 (c2 INT) ENGINE=MyISAM; Query OK, 0 rows affected (0,023 sec) MariaDB [test]> CREATE TABLE t3 (c3 INT) ENGINE=MyISAM; Query OK, 0 rows affected (0,021 sec) MariaDB [test]> EXPLAIN EXTENDED UPDATE t3 SET c3 = -> ( SELECT COUNT(d1.c1) FROM ( SELECT a11.c1 FROM t1 AS a11 -> STRAIGHT_JOIN t2 AS a21 ON a21.c2 = a11.c1 JOIN t1 AS a12 -> ON a12.c1 = a11.c1 ) d1 ); +------+-------------+-------+------+---------------+------+---------+------+------+----------+--------------------------------+ | id | select_type | table | type | possible_keys | key | key_len | ref | rows | filtered | Extra | +------+-------------+-------+------+---------------+------+---------+------+------+----------+--------------------------------+ | 1 | PRIMARY | t3 | ALL | NULL | NULL | NULL | NULL | 0 | 100.00 | | | 2 | SUBQUERY | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | Impossible WHERE noticed after reading const tables +------+-------------+-------+------+---------------+------+---------+------+------+----------+--------------------------------+ 2 rows in set (0,002 sec) MariaDB [test]> PREPARE stmt FROM -> EXPLAIN EXTENDED UPDATE t3 SET c3 = -> ( SELECT COUNT(d1.c1) FROM ( SELECT a11.c1 FROM t1 AS a11 -> STRAIGHT_JOIN t2 AS a21 ON a21.c2 = a11.c1 JOIN t1 AS a12 -> ON a12.c1 = a11.c1 ) d1 ); Query OK, 0 rows affected (0,000 sec) Statement prepared MariaDB [test]> EXECUTE stmt; +------+-------------+-------+------+---------------+------+---------+------+------+----------+--------------------------------+ | id | select_type | table | type | possible_keys | key | key_len | ref | rows | filtered | Extra | +------+-------------+-------+------+---------------+------+---------+------+------+----------+--------------------------------+ | 1 | PRIMARY | t3 | ALL | NULL | NULL | NULL | NULL | 0 | 100.00 | | | 2 | SUBQUERY | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | no matching row in const table | +------+-------------+-------+------+---------------+------+---------+------+------+----------+--------------------------------+ 2 rows in set (0,000 sec) The reason by that different result sets are produced is that on execution of the statement 'EXECUTE stmt' the flag SELECT_DESCRIBE not set in the data member SELECT_LEX::options for instances of SELECT_LEX that correspond to subqueries used in the UPDTAE/DELETE statements. Initially, these flags were set on parsing the statement PREPARE stmt FROM "EXPLAIN EXTENDED UPDATE t3 SET ..." but latter they were reset before starting real execution of the parsed query during handling the statement 'EXECUTE stmt'; So, to fix the issue the functions mysql_update()/mysql_delete() have been modified to set the flag SELECT_DESCRIBE forcibly in the data member SELECT_LEX::options for the primary SELECT_LEX of the UPDATE/DELETE statement. --- mysql-test/r/ps.result | 18 ++++++++++++++++++ mysql-test/t/ps.test | 14 ++++++++++++++ sql/sql_base.cc | 17 +++++++++++++++++ sql/sql_base.h | 2 ++ sql/sql_delete.cc | 2 ++ sql/sql_update.cc | 2 ++ 6 files changed, 55 insertions(+) diff --git a/mysql-test/r/ps.result b/mysql-test/r/ps.result index 4ee72f10785..f1f779ef03f 100644 --- a/mysql-test/r/ps.result +++ b/mysql-test/r/ps.result @@ -5456,5 +5456,23 @@ Note 1003 select (select 1 from `test`.`t2` where 0) AS `(SELECT 1 FROM t2 WHERE DEALLOCATE PREPARE stmt; DROP TABLE t1, t2; # +# MDEV-25576: The statement EXPLAIN running as regular statement and +# as prepared statement produces different results for +# UPDATE with subquery +# +CREATE TABLE t1 (c1 INT KEY) ENGINE=MyISAM; +CREATE TABLE t2 (c2 INT) ENGINE=MyISAM; +CREATE TABLE t3 (c3 INT) ENGINE=MyISAM; +EXPLAIN EXTENDED UPDATE t3 SET c3 = ( SELECT COUNT(d1.c1) FROM ( SELECT a11.c1 FROM t1 AS a11 STRAIGHT_JOIN t2 AS a21 ON a21.c2 = a11.c1 JOIN t1 AS a12 ON a12.c1 = a11.c1 ) d1 ); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t3 ALL NULL NULL NULL NULL 0 100.00 +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +PREPARE stmt FROM "EXPLAIN EXTENDED UPDATE t3 SET c3 = ( SELECT COUNT(d1.c1) FROM ( SELECT a11.c1 FROM t1 AS a11 STRAIGHT_JOIN t2 AS a21 ON a21.c2 = a11.c1 JOIN t1 AS a12 ON a12.c1 = a11.c1 ) d1 )"; +EXECUTE stmt; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t3 ALL NULL NULL NULL NULL 0 100.00 +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +DEALLOCATE PREPARE stmt; +DROP TABLE t1, t2, t3; # End of 10.2 tests # diff --git a/mysql-test/t/ps.test b/mysql-test/t/ps.test index 5b86f82a9cb..2e7b43ad748 100644 --- a/mysql-test/t/ps.test +++ b/mysql-test/t/ps.test @@ -4946,6 +4946,20 @@ SHOW WARNINGS; DEALLOCATE PREPARE stmt; DROP TABLE t1, t2; + --echo # +--echo # MDEV-25576: The statement EXPLAIN running as regular statement and +--echo # as prepared statement produces different results for +--echo # UPDATE with subquery +--echo # +CREATE TABLE t1 (c1 INT KEY) ENGINE=MyISAM; +CREATE TABLE t2 (c2 INT) ENGINE=MyISAM; +CREATE TABLE t3 (c3 INT) ENGINE=MyISAM; +EXPLAIN EXTENDED UPDATE t3 SET c3 = ( SELECT COUNT(d1.c1) FROM ( SELECT a11.c1 FROM t1 AS a11 STRAIGHT_JOIN t2 AS a21 ON a21.c2 = a11.c1 JOIN t1 AS a12 ON a12.c1 = a11.c1 ) d1 ); +PREPARE stmt FROM "EXPLAIN EXTENDED UPDATE t3 SET c3 = ( SELECT COUNT(d1.c1) FROM ( SELECT a11.c1 FROM t1 AS a11 STRAIGHT_JOIN t2 AS a21 ON a21.c2 = a11.c1 JOIN t1 AS a12 ON a12.c1 = a11.c1 ) d1 )"; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +DROP TABLE t1, t2, t3; + --echo # End of 10.2 tests --echo # diff --git a/sql/sql_base.cc b/sql/sql_base.cc index c9a34221544..11f4cb9890b 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -8794,6 +8794,23 @@ int dynamic_column_error_message(enum_dyncol_func_result rc) return rc; } + +/** + Turn on the SELECT_DESCRIBE flag for the primary SELECT_LEX of the statement + being processed in case the statement is EXPLAIN UPDATE/DELETE. + + @param lex current LEX +*/ + +void promote_select_describe_flag_if_needed(LEX *lex) +{ + if (lex->describe) + { + lex->select_lex.options |= SELECT_DESCRIBE; + } +} + + /** @} (end of group Data_Dictionary) */ diff --git a/sql/sql_base.h b/sql/sql_base.h index b67341bcbda..5674e8378c5 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -516,6 +516,8 @@ bool extend_table_list(THD *thd, TABLE_LIST *tables, Prelocking_strategy *prelocking_strategy, bool has_prelocking_list); +void promote_select_describe_flag_if_needed(LEX *lex); + /** A context of open_tables() function, used to recover from a failed open_table() or open_routine() attempt. diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index e2bf2d727b1..a26e15431a6 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -274,6 +274,8 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, query_plan.table= table; query_plan.updating_a_view= MY_TEST(table_list->view); + promote_select_describe_flag_if_needed(thd->lex); + if (mysql_prepare_delete(thd, table_list, select_lex->with_wild, select_lex->item_list, &conds)) DBUG_RETURN(TRUE); diff --git a/sql/sql_update.cc b/sql/sql_update.cc index ec27ccda778..b7ef7c2a0db 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -342,6 +342,8 @@ int mysql_update(THD *thd, want_privilege= (table_list->view ? UPDATE_ACL : table_list->grant.want_privilege); #endif + promote_select_describe_flag_if_needed(thd->lex); + if (mysql_prepare_update(thd, table_list, &conds, order_num, order)) DBUG_RETURN(1); From d3c77e08ae0a16182627701d685a5c3762112ebd Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 31 May 2021 12:27:47 +0200 Subject: [PATCH 22/30] MDEV-20556 Remove references to "xtrabackup" and "innobackupex" in mariabackup --help --- extra/mariabackup/innobackupex.cc | 86 +++++++++++---------------- extra/mariabackup/xtrabackup.cc | 98 +++---------------------------- 2 files changed, 41 insertions(+), 143 deletions(-) diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index fefd81dc1be..c4c8ec7f17b 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -214,7 +214,7 @@ ibx_mode_t ibx_mode = IBX_MODE_BACKUP; static struct my_option ibx_long_options[] = { - {"version", 'v', "print xtrabackup version information", + {"version", 'v', "print version information", (uchar *) &opt_ibx_version, (uchar *) &opt_ibx_version, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -257,12 +257,11 @@ static struct my_option ibx_long_options[] = (uchar *) &opt_ibx_slave_info, (uchar *) &opt_ibx_slave_info, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"incremental", OPT_INCREMENTAL, "This option tells xtrabackup to " - "create an incremental backup, rather than a full one. It is passed " - "to the xtrabackup child process. When this option is specified, " + {"incremental", OPT_INCREMENTAL, + "Create an incremental backup, rather than a full one. When this option is specified, " "either --incremental-lsn or --incremental-basedir can also be given. " - "If neither option is given, option --incremental-basedir is passed " - "to xtrabackup by default, set to the first timestamped backup " + "If neither option is given, option --incremental-basedir is used " + "by default, set to the first timestamped backup " "directory in the backup base directory.", (uchar *) &opt_ibx_incremental, (uchar *) &opt_ibx_incremental, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -377,14 +376,14 @@ static struct my_option ibx_long_options[] = {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME, "This option specifies the name of the backup series stored in the " "PERCONA_SCHEMA.xtrabackup_history history record to base an " - "incremental backup on. Xtrabackup will search the history table " + "incremental backup on. Backup will search the history table " "looking for the most recent (highest innodb_to_lsn), successful " "backup in the series and take the to_lsn value to use as the " "starting lsn for the incremental backup. This will be mutually " "exclusive with --incremental-history-uuid, --incremental-basedir " "and --incremental-lsn. If no valid lsn can be found (no series by " - "that name, no successful backups by that name) xtrabackup will " - "return with an error. It is used with the --incremental option.", + "that name, no successful backups by that name), " + "an error will be returned. It is used with the --incremental option.", (uchar*) &opt_ibx_incremental_history_name, (uchar*) &opt_ibx_incremental_history_name, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -394,8 +393,8 @@ static struct my_option ibx_long_options[] = "stored in the PERCONA_SCHEMA.xtrabackup_history to base an " "incremental backup on. --incremental-history-name, " "--incremental-basedir and --incremental-lsn. If no valid lsn can be " - "found (no success record with that uuid) xtrabackup will return " - "with an error. It is used with the --incremental option.", + "found (no success record with that uuid), an error will be returned." + " It is used with the --incremental option.", (uchar*) &opt_ibx_incremental_history_uuid, (uchar*) &opt_ibx_incremental_history_uuid, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -424,7 +423,7 @@ static struct my_option ibx_long_options[] = {"include", OPT_INCLUDE, "This option is a regular expression to be matched against table " "names in databasename.tablename format. It is passed directly to " - "xtrabackup's --tables option. See the xtrabackup documentation for " + "--tables option. See the documentation for " "details.", (uchar*) &opt_ibx_include, (uchar*) &opt_ibx_include, 0, GET_STR, @@ -474,12 +473,6 @@ static struct my_option ibx_long_options[] = (uchar*) &opt_ibx_lock_wait_threshold, 0, GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, - {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK, - "This is a debug-only option used by the XtraBackup test suite.", - (uchar*) &opt_ibx_debug_sleep_before_unlock, - (uchar*) &opt_ibx_debug_sleep_before_unlock, 0, GET_UINT, - REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, "How many seconds --safe-slave-backup should wait for " "Slave_open_temp_tables to become zero. (default 300)", @@ -492,22 +485,20 @@ static struct my_option ibx_long_options[] = We put them here with only purpose for them to showup in innobackupex --help output */ - {"close_files", OPT_CLOSE_FILES, "Do not keep files opened. This " - "option is passed directly to xtrabackup. Use at your own risk.", + {"close_files", OPT_CLOSE_FILES, "Do not keep files opened." + " Use at your own risk.", (uchar*) &ibx_xb_close_files, (uchar*) &ibx_xb_close_files, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"compress", OPT_COMPRESS, "This option instructs xtrabackup to " - "compress backup copies of InnoDB data files. It is passed directly " - "to the xtrabackup child process. Try 'xtrabackup --help' for more " - "details.", (uchar*) &ibx_xtrabackup_compress_alg, + {"compress", OPT_COMPRESS, "This option instructs backup to " + "compress backup copies of InnoDB data files." + , (uchar*) &ibx_xtrabackup_compress_alg, (uchar*) &ibx_xtrabackup_compress_alg, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, {"compress-threads", OPT_COMPRESS_THREADS, "This option specifies the number of worker threads that will be used " - "for parallel compression. It is passed directly to the xtrabackup " - "child process. Try 'xtrabackup --help' for more details.", + "for parallel compression.", (uchar*) &ibx_xtrabackup_compress_threads, (uchar*) &ibx_xtrabackup_compress_threads, 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, @@ -518,17 +509,15 @@ static struct my_option ibx_long_options[] = (uchar*) &ibx_xtrabackup_compress_chunk_size, 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, - {"export", OPT_EXPORT, "This option is passed directly to xtrabackup's " - "--export option. It enables exporting individual tables for import " - "into another server. See the xtrabackup documentation for details.", + {"export", OPT_EXPORT, " enables exporting individual tables for import " + "into another server.", (uchar*) &ibx_xtrabackup_export, (uchar*) &ibx_xtrabackup_export, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"extra-lsndir", OPT_EXTRA_LSNDIR, "This option specifies the " "directory in which to save an extra copy of the " "\"xtrabackup_checkpoints\" file. The option accepts a string " - "argument. It is passed directly to xtrabackup's --extra-lsndir " - "option. See the xtrabackup documentation for details.", + "argument.", (uchar*) &ibx_xtrabackup_extra_lsndir, (uchar*) &ibx_xtrabackup_extra_lsndir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -550,7 +539,7 @@ static struct my_option ibx_long_options[] = 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"incremental-force-scan", OPT_INCREMENTAL_FORCE_SCAN, - "This options tells xtrabackup to perform full scan of data files " + "Perform full scan of data files " "for taking an incremental backup even if full changed page bitmap " "data is available to enable the backup without the full scan.", (uchar*)&ibx_xtrabackup_incremental_force_scan, @@ -576,10 +565,8 @@ static struct my_option ibx_long_options[] = 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"parallel", OPT_PARALLEL, "On backup, this option specifies the " - "number of threads the xtrabackup child process should use to back " - "up files concurrently. The option accepts an integer argument. It " - "is passed directly to xtrabackup's --parallel option. See the " - "xtrabackup documentation for details.", + "number of threads to use to back " + "up files concurrently. The option accepts an integer argument.", (uchar*) &ibx_xtrabackup_parallel, (uchar*) &ibx_xtrabackup_parallel, 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, @@ -587,23 +574,21 @@ static struct my_option ibx_long_options[] = {"stream", OPT_STREAM, "This option specifies the format in which to " "do the streamed backup. The option accepts a string argument. The " "backup will be done to STDOUT in the specified format. Currently, " - "the only supported formats are tar and mbstream/xbstream. This " - "option is passed directly to xtrabackup's --stream option.", + "the only supported formats are tar and mbstream/xbstream.", (uchar*) &ibx_xtrabackup_stream_str, (uchar*) &ibx_xtrabackup_stream_str, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"tables-file", OPT_TABLES_FILE, "This option specifies the file in " "which there are a list of names of the form database. The option " - "accepts a string argument.table, one per line. The option is passed " - "directly to xtrabackup's --tables-file option.", + "accepts a string argument.table, one per line.", (uchar*) &ibx_xtrabackup_tables_file, (uchar*) &ibx_xtrabackup_tables_file, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"throttle", OPT_THROTTLE, "This option specifies a number of I/O " "operations (pairs of read+write) per second. It accepts an integer " - "argument. It is passed directly to xtrabackup's --throttle option.", + "argument.", (uchar*) &ibx_xtrabackup_throttle, (uchar*) &ibx_xtrabackup_throttle, 0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0}, @@ -615,11 +600,10 @@ static struct my_option ibx_long_options[] = 0, 0, 0, 0, 0, 0}, {"use-memory", OPT_USE_MEMORY, "This option accepts a string argument " - "that specifies the amount of memory in bytes for xtrabackup to use " + "that specifies the amount of memory in bytes to use " "for crash recovery while preparing a backup. Multiples are supported " "providing the unit (e.g. 1MB, 1GB). It is used only with the option " - "--apply-log. It is passed directly to xtrabackup's --use-memory " - "option. See the xtrabackup documentation for details.", + "--apply-log.", (uchar*) &ibx_xtrabackup_use_memory, (uchar*) &ibx_xtrabackup_use_memory, 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, @@ -641,7 +625,7 @@ static struct my_option ibx_long_options[] = static void usage(void) { - puts("Open source backup tool for InnoDB and XtraDB\n\ + puts("Open source backup tool\n\ \n\ Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ @@ -692,7 +676,7 @@ innobackupex [--decompress]\n\ \n\ DESCRIPTION\n\ \n\ -The first command line above makes a hot backup of a MySQL database.\n\ +The first command line above makes a hot backup of a database.\n\ By default it creates a backup directory (named by the current date\n\ and time) in the given backup root directory. With the --no-timestamp\n\ option it does not create a time-stamped backup directory, but it puts\n\ @@ -702,22 +686,18 @@ indexes in all databases or in all of the databases specified with the\n\ --databases option. The created backup contains .frm, .MRG, .MYD,\n\ .MYI, .MAD, .MAI, .TRG, .TRN, .ARM, .ARZ, .CSM, CSV, .opt, .par, and\n\ InnoDB data and log files. The MY.CNF options file defines the\n\ -location of the database. This command connects to the MySQL server\n\ -using the mysql client program, and runs xtrabackup as a child\n\ -process.\n\ +location of the database.\n\ \n\ The --apply-log command prepares a backup for starting a MySQL\n\ server on the backup. This command recovers InnoDB data files as specified\n\ in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/ib_logfile0,\n\ and creates new InnoDB log files as specified in BACKUP-DIR/backup-my.cnf.\n\ -The BACKUP-DIR should be the path to a backup directory created by\n\ -xtrabackup. This command runs xtrabackup as a child process, but it does not \n\ -connect to the database server.\n\ +The BACKUP-DIR should be the path to a backup directory\n\ \n\ The --copy-back command copies data, index, and log files\n\ from the backup directory back to their original locations.\n\ The MY.CNF options file defines the original location of the database.\n\ -The BACKUP-DIR is the path to a backup directory created by xtrabackup.\n\ +The BACKUP-DIR is the path to a backup directory.\n\ \n\ The --move-back command is similar to --copy-back with the only difference that\n\ it moves files to their original locations rather than copies them. As this\n\ diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index d4e7114f635..dc6227b6b7b 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -267,8 +267,6 @@ it every INNOBASE_WAKE_INTERVAL'th step. */ ulong innobase_active_counter = 0; -static char *xtrabackup_debug_sync = NULL; - my_bool xtrabackup_incremental_force_scan = FALSE; /* @@ -1059,7 +1057,7 @@ enum options_xtrabackup struct my_option xb_client_options[]= { {"verbose", 'V', "display verbose output", (G_PTR *) &verbose, (G_PTR *) &verbose, 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0}, - {"version", 'v', "print xtrabackup version information", + {"version", 'v', "print version information", (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory", @@ -1356,8 +1354,8 @@ struct my_option xb_client_options[]= { "starting lsn for the incremental backup. This will be mutually " "exclusive with --incremental-history-uuid, --incremental-basedir " "and --incremental-lsn. If no valid lsn can be found (no series by " - "that name, no successful backups by that name) xtrabackup will " - "return with an error. It is used with the --incremental option.", + "that name, no successful backups by that name), an error will be returned." + " It is used with the --incremental option.", (uchar *) &opt_incremental_history_name, (uchar *) &opt_incremental_history_name, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -1367,8 +1365,8 @@ struct my_option xb_client_options[]= { "stored in the PERCONA_SCHEMA.xtrabackup_history to base an " "incremental backup on. --incremental-history-name, " "--incremental-basedir and --incremental-lsn. If no valid lsn can be " - "found (no success record with that uuid) xtrabackup will return " - "with an error. It is used with the --incremental option.", + "found (no success record with that uuid), an error will be returned." + " It is used with the --incremental option.", (uchar *) &opt_incremental_history_uuid, (uchar *) &opt_incremental_history_uuid, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -1425,11 +1423,6 @@ struct my_option xb_client_options[]= { (uchar *) &opt_lock_wait_threshold, (uchar *) &opt_lock_wait_threshold, 0, GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, - {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK, - "This is a debug-only option used by the XtraBackup test suite.", - (uchar *) &opt_debug_sleep_before_unlock, - (uchar *) &opt_debug_sleep_before_unlock, 0, GET_UINT, REQUIRED_ARG, 0, 0, - 0, 0, 0, 0}, {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, "How many seconds --safe-slave-backup should wait for " @@ -1439,9 +1432,9 @@ struct my_option xb_client_options[]= { 0, 0, 0, 0, 0}, {"binlog-info", OPT_BINLOG_INFO, - "This option controls how XtraBackup should retrieve server's binary log " + "This option controls how backup should retrieve server's binary log " "coordinates corresponding to the backup. Possible values are OFF, ON, " - "LOCKLESS and AUTO. See the XtraBackup manual for more information", + "LOCKLESS and AUTO.", &opt_binlog_info, &opt_binlog_info, &binlog_info_typelib, GET_ENUM, OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0}, @@ -1615,13 +1608,6 @@ struct my_option xb_server_options[] = &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, #endif -#ifndef __WIN__ - {"debug-sync", OPT_XTRA_DEBUG_SYNC, - "Debug sync point. This is only used by the xtrabackup test suite", - (G_PTR*) &xtrabackup_debug_sync, - (G_PTR*) &xtrabackup_debug_sync, - 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, -#endif {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM, "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, " @@ -1665,7 +1651,7 @@ struct my_option xb_server_options[] = REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0}, {"lock-ddl-per-table", OPT_LOCK_DDL_PER_TABLE, "Lock DDL for each table " - "before xtrabackup starts to copy it and until the backup is completed.", + "before backup starts to copy it and until the backup is completed.", (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -1696,60 +1682,6 @@ struct my_option xb_server_options[] = uint xb_server_options_count = array_elements(xb_server_options); -#ifndef __WIN__ -static int debug_sync_resumed; - -static void sigcont_handler(int sig); - -static void sigcont_handler(int sig __attribute__((unused))) -{ - debug_sync_resumed= 1; -} -#endif - -static inline -void -debug_sync_point(const char *name) -{ -#ifndef __WIN__ - FILE *fp; - pid_t pid; - char pid_path[FN_REFLEN]; - - if (xtrabackup_debug_sync == NULL) { - return; - } - - if (strcmp(xtrabackup_debug_sync, name)) { - return; - } - - pid = getpid(); - - snprintf(pid_path, sizeof(pid_path), "%s/xtrabackup_debug_sync", - xtrabackup_target_dir); - fp = fopen(pid_path, "w"); - if (fp == NULL) { - die("Can't open open %s", pid_path); - } - fprintf(fp, "%u\n", (uint) pid); - fclose(fp); - - msg("mariabackup: DEBUG: Suspending at debug sync point '%s'. " - "Resume with 'kill -SIGCONT %u'.", name, (uint) pid); - - debug_sync_resumed= 0; - kill(pid, SIGSTOP); - while (!debug_sync_resumed) { - sleep(1); - } - - /* On resume */ - msg("mariabackup: DEBUG: removing the pid file."); - my_delete(pid_path, MYF(MY_WME)); -#endif -} - static std::set tables_for_export; @@ -3109,8 +3041,6 @@ static bool xtrabackup_copy_logfile(bool last = false) log_copy_scanned_lsn = start_lsn; pthread_cond_broadcast(&scanned_lsn_cond); pthread_mutex_unlock(&backup_mutex); - - debug_sync_point("xtrabackup_copy_logfile_pause"); return(false); } @@ -3242,8 +3172,6 @@ data_copy_thread_func( */ my_thread_init(); - debug_sync_point("data_copy_thread_func"); - while ((node = datafiles_iter_next(ctxt->it)) != NULL) { DBUG_MARIABACKUP_EVENT("before_copy", node->space->name); DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", node->space->name, @@ -3774,8 +3702,6 @@ xb_load_tablespaces() if (err != DB_SUCCESS) { return(err); } - - debug_sync_point("xtrabackup_load_tablespaces_pause"); DBUG_MARIABACKUP_EVENT("after_load_tablespaces", 0); return(DB_SUCCESS); } @@ -4660,8 +4586,6 @@ fail_before_log_copying_thread_start: if (!flush_changed_page_bitmaps()) { goto fail; } - debug_sync_point("xtrabackup_suspend_at_start"); - ut_a(xtrabackup_parallel > 0); @@ -6881,12 +6805,6 @@ static int main_low(char** argv) } } -#ifndef __WIN__ - if (xtrabackup_debug_sync) { - signal(SIGCONT, sigcont_handler); - } -#endif - /* --backup */ if (xtrabackup_backup && !xtrabackup_backup_func()) { return(EXIT_FAILURE); From 601eb41183a6c2a646ce221da675f157181fd57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 27 May 2021 16:17:43 +0300 Subject: [PATCH 23/30] Cleanup: deduplicate code --- storage/innobase/buf/buf0flu.cc | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index fb3a9687cfc..7d1136043b7 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -2361,14 +2361,8 @@ void buf_flush_sync() { const ulint n_flushed= buf_flush_lists(srv_max_io_capacity, LSN_MAX); buf_flush_wait_batch_end_acquiring_mutex(false); - if (!n_flushed) - { - mysql_mutex_lock(&buf_pool.flush_list_mutex); - const auto len= UT_LIST_GET_LEN(buf_pool.flush_list); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!len) - return; - } + if (!n_flushed && !buf_flush_list_length()) + return; } } From 6ca065468f16fcd9ecc4ab8a874aafc778154b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 27 May 2021 16:46:11 +0300 Subject: [PATCH 24/30] MDEV-19514 fixup: Optimize ibuf_merge_or_delete_for_page() calls --- storage/innobase/buf/buf0buf.cc | 3 ++- storage/innobase/ibuf/ibuf0ibuf.cc | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 0079f6582ab..4b0e1edd6c9 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -3812,7 +3812,8 @@ loop: /* Delete possible entries for the page from the insert buffer: such can exist if the page belonged to an index which was dropped */ - if (!recv_recovery_is_on()) + if (page_id < page_id_t{SRV_SPACE_ID_UPPER_BOUND, 0} && + !recv_recovery_is_on()) ibuf_merge_or_delete_for_page(nullptr, page_id, zip_size); static_assert(FIL_PAGE_PREV + 4 == FIL_PAGE_NEXT, "adjacent"); diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index a3d6ed26cda..3c7f2de2a67 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -4203,6 +4203,10 @@ subsequently was dropped. void ibuf_merge_or_delete_for_page(buf_block_t *block, const page_id_t page_id, ulint zip_size) { + if (trx_sys_hdr_page(page_id)) { + return; + } + btr_pcur_t pcur; #ifdef UNIV_IBUF_DEBUG ulint volume = 0; @@ -4217,11 +4221,8 @@ void ibuf_merge_or_delete_for_page(buf_block_t *block, const page_id_t page_id, ut_ad(!block || page_id == block->page.id()); ut_ad(!block || block->page.state() == BUF_BLOCK_FILE_PAGE); ut_ad(!block || block->page.status == buf_page_t::NORMAL); - - if (trx_sys_hdr_page(page_id) - || fsp_is_system_temporary(page_id.space())) { - return; - } + ut_ad(!trx_sys_hdr_page(page_id)); + ut_ad(page_id < page_id_t(SRV_SPACE_ID_UPPER_BOUND, 0)); const ulint physical_size = zip_size ? zip_size : srv_page_size; From 139333a6cc5841bbc01ee8f73acef2689274d794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 31 May 2021 15:44:11 +0300 Subject: [PATCH 25/30] MDEV-25745: Not applying INSERT_REUSE_REDUNDANT page_apply_insert_redundant(): Correct a condition that would occasionally fail when recovering changes for the change buffer tree (where extra_size and data_size can vary wildly). This was broken in commit 138cbec5f2300bd5b401e83802642c1806264992 (MDEV-21724). --- storage/innobase/page/page0cur.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 025d4b9b967..f75d1a8be77 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -2443,7 +2443,7 @@ corrupted: if (UNIV_UNLIKELY(free_rec - fextra_size < heap_bot)) goto corrupted; const ulint fdata_size= rec_get_data_size_old(free_rec); - if (UNIV_UNLIKELY(free_rec + data_size > heap_top)) + if (UNIV_UNLIKELY(free_rec + fdata_size > heap_top)) goto corrupted; if (UNIV_UNLIKELY(extra_size + data_size > fextra_size + fdata_size)) goto corrupted; From 2fb4407827ecd6cbf52e210a8d9370b4560ddd5b Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Sat, 29 May 2021 19:54:25 +0200 Subject: [PATCH 26/30] MDEV-25818: RSYNC SST failed due to busy port This commit reduces the likelihood of getting a busy port on quick restarts with rsync SST (problem MDEV-25818) and fixes a number of other flaws in SST scripts, adds new functionality, and also synchronizes the xtrabackup-v2 script with the mariabackup script (the latter applies only to the 10.2 branch): 1) SST via rsync: rsync and stunnel does not always get the right time to complete by correctly handling SIGTERM. These utilities are now given more time to complete normally (via normal SIGTERM processing) before we move on to using "kill -9"; 2) SST via rsync: attempts to terminate an rsync or stunnel process (via "kill" utility) are only made if it did not terminated on its own; 3) SST via rsync: if a combination of stunnel and rsync is used, then we need to wait for both utilities to finish or stop, not just one of them; 4) The config file and pid file for stunnel are now deleted after successful completion of SST on the donor node; 5) The configs and pid files from rsync and stunnel should not be deleted unless these utilities succeed (or are sucessfully terminated) on the joiner node; 6) The configs and pid files now excluded from transfer via rsync; 7) Spaces in paths are now valid for config files as well (when used with SST via rsync or mariabackup / xtrabackup[-v2]); 8) SST via mariabackup: added preliminary verification of keys and certificates that are used when establishing a connection using SSL (to avoid long timeouts and improve diagnostics) - by analogy with how it is done for the xtrabackup-v2 (plus check for CA file), while that check is skipped if the user does not have openssl installed (or does not have diff utility); 9) Added backup-threads= configuration option which adds "--parallel=" for mariabackup / xtrabackup at backup and move-back stages; 10) Added encrypt-threads and encrypt-chunk-size configuration options for xbcrypt management (when xbcrypt is used); 11) Small optimization: checking the socat version and adding a file with parameters for 2048-bit Diffie-Hellman (if necessary) is done only if the user has not specified "dhparam=" in the "sockopt" option value; 12) SST via rsync now supports "backup-threads" configuration option (in server-related sections or in the "[sst]"); 13) Determining the number of available processors is now supported for FreeBSD + mariabackup/xtrabackup: before that we might have problems with "--compact" (rebuild indexes) or qpress on FreeBSD; 14) The check_pid() function should not raise an error state in the rare cases when the pid file was created, but it is empty, or if it is deleted right during the check, or when zero is read from the pid file; 15) Iproved templates that are used to check if a requested socket is "listening" when using the ss utility; 16) Shortened some other templates for socket state utilities; 17) Temporary files created by mariabackup / xtrabackup are moved to a separate subdirectory inside tmpdir (so they don't get mixed with other temporary files, which can make debugging more difficult); 18) 10.2 only: the script for SST via xtrabackup-v2 has been brought in full compliance with all the bugfixes made for mariabackup (as it previously contained many flaws compared to the updated script for mariabackup). --- .../t/galera_autoinc_sst_xtrabackup.cnf | 1 - .../suite/galera/t/galera_ist_mariabackup.cnf | 1 - ...lera_ist_mariabackup_innodb_flush_logs.cnf | 1 - .../suite/galera/t/galera_ist_mysqldump.cnf | 2 - .../galera/t/galera_ist_xtrabackup-v2.cnf | 2 - .../suite/galera/t/galera_sst_mariabackup.cnf | 2 +- .../t/galera_sst_mariabackup_qpress.cnf | 1 + .../galera_sst_mariabackup_table_options.cnf | 2 +- .../suite/galera/t/galera_sst_rsync.cnf | 1 - .../suite/galera/t/galera_sst_rsync2.cnf | 1 - .../galera/t/galera_sst_rsync_data_dir.cnf | 3 + scripts/wsrep_sst_common.sh | 361 +++++- scripts/wsrep_sst_mariabackup.sh | 292 ++--- scripts/wsrep_sst_mysqldump.sh | 4 +- scripts/wsrep_sst_rsync.sh | 263 +++-- scripts/wsrep_sst_xtrabackup-v2.sh | 1052 ++++++++++------- scripts/wsrep_sst_xtrabackup.sh | 23 +- 17 files changed, 1182 insertions(+), 830 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf b/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf index 7da7dd4e73b..2e25ecae317 100644 --- a/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf +++ b/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf @@ -10,4 +10,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore [mysqld.2] wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' - diff --git a/mysql-test/suite/galera/t/galera_ist_mariabackup.cnf b/mysql-test/suite/galera/t/galera_ist_mariabackup.cnf index 75dff78149d..3dc62546591 100644 --- a/mysql-test/suite/galera/t/galera_ist_mariabackup.cnf +++ b/mysql-test/suite/galera/t/galera_ist_mariabackup.cnf @@ -9,4 +9,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' [mysqld.2] wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true' - diff --git a/mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf b/mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf index 1542376b2b8..7979b23e085 100644 --- a/mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf +++ b/mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf @@ -11,4 +11,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' [mysqld.2] wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true' - diff --git a/mysql-test/suite/galera/t/galera_ist_mysqldump.cnf b/mysql-test/suite/galera/t/galera_ist_mysqldump.cnf index 357c8678658..807829f047f 100644 --- a/mysql-test/suite/galera/t/galera_ist_mysqldump.cnf +++ b/mysql-test/suite/galera/t/galera_ist_mysqldump.cnf @@ -8,5 +8,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' [mysqld.2] wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true' - - diff --git a/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf b/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf index 52df3814129..5b35701aeab 100644 --- a/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf +++ b/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf @@ -5,10 +5,8 @@ wsrep_sst_method=xtrabackup-v2 wsrep_sst_auth=root: innodb_safe_truncate=OFF - [mysqld.1] wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' [mysqld.2] wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true' - diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup.cnf index 336296e9bfe..4110192b947 100644 --- a/mysql-test/suite/galera/t/galera_sst_mariabackup.cnf +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup.cnf @@ -13,4 +13,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore [sst] transferfmt=@ENV.MTR_GALERA_TFMT -streamfmt=xbstream +streamfmt=mbstream diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf index 25a9d5fc0b2..f58d22cba52 100644 --- a/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf @@ -10,3 +10,4 @@ transferfmt=@ENV.MTR_GALERA_TFMT compress=quicklz compress-threads=2 compress-chunk-size=32768 +backup-threads=2 diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf index 5e6913d4d2b..13cb1aa5366 100644 --- a/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf @@ -15,4 +15,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore [sst] transferfmt=@ENV.MTR_GALERA_TFMT -streamfmt=xbstream +streamfmt=mbstream diff --git a/mysql-test/suite/galera/t/galera_sst_rsync.cnf b/mysql-test/suite/galera/t/galera_sst_rsync.cnf index 93981d9daa7..29d3a1e6219 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync.cnf +++ b/mysql-test/suite/galera/t/galera_sst_rsync.cnf @@ -8,4 +8,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore [mysqld.2] wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' - diff --git a/mysql-test/suite/galera/t/galera_sst_rsync2.cnf b/mysql-test/suite/galera/t/galera_sst_rsync2.cnf index 34e67c66403..0159596f99b 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync2.cnf +++ b/mysql-test/suite/galera/t/galera_sst_rsync2.cnf @@ -12,4 +12,3 @@ log_bin_index=@ENV.MYSQLTEST_VARDIR/tmp/server1_binlog_index.index wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' log_bin=@ENV.MYSQLTEST_VARDIR/server2_binlog log_bin_index=@ENV.MYSQLTEST_VARDIR/tmp/server2_binlog_index.index - diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf index afe9796a11a..4cc3442bd87 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf +++ b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf @@ -9,3 +9,6 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore [mysqld.2] innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test_2 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' + +[sst] +backup_threads=2 diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 261912b86ad..05944ef6035 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -29,7 +29,9 @@ WSREP_SST_OPT_USER="${WSREP_SST_OPT_USER:-}" WSREP_SST_OPT_PSWD="${WSREP_SST_OPT_PSWD:-}" WSREP_SST_OPT_REMOTE_AUTH="${WSREP_SST_OPT_REMOTE_AUTH:-}" WSREP_SST_OPT_DEFAULT="" +WSREP_SST_OPT_DEFAULTS="" WSREP_SST_OPT_EXTRA_DEFAULT="" +WSREP_SST_OPT_EXTRA_DEFAULTS="" WSREP_SST_OPT_SUFFIX_DEFAULT="" WSREP_SST_OPT_SUFFIX_VALUE="" WSREP_SST_OPT_MYSQLD="" @@ -152,10 +154,12 @@ case "$1" in ;; '--defaults-file') readonly WSREP_SST_OPT_DEFAULT="$1=$2" + readonly WSREP_SST_OPT_DEFAULTS="$1='$2'" shift ;; '--defaults-extra-file') readonly WSREP_SST_OPT_EXTRA_DEFAULT="$1=$2" + readonly WSREP_SST_OPT_EXTRA_DEFAULTS="$1='$2'" shift ;; '--defaults-group-suffix') @@ -611,24 +615,54 @@ else MYSQLDUMP="$(command -v mysqldump)" fi +wsrep_log() +{ + # echo everything to stderr so that it gets into common error log + # deliberately made to look different from the rest of the log + local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)" + echo "WSREP_SST: $* ($tst)" >&2 +} + +wsrep_log_error() +{ + wsrep_log "[ERROR] $*" +} + +wsrep_log_warning() +{ + wsrep_log "[WARNING] $*" +} + +wsrep_log_info() +{ + wsrep_log "[INFO] $*" +} + if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then MY_PRINT_DEFAULTS="$SCRIPTS_DIR/my_print_defaults" elif [ -x "$EXTRA_DIR/my_print_defaults" ]; then MY_PRINT_DEFAULTS="$EXTRA_DIR/my_print_defaults" else MY_PRINT_DEFAULTS="$(command -v my_print_defaults)" + if [ -z "$MY_PRINT_DEFAULTS" ]; then + wsrep_log_error "my_print_defaults not found in path" + exit 2 + fi fi +readonly MY_PRINT_DEFAULTS + +wsrep_defaults="$WSREP_SST_OPT_DEFAULTS" +wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULTS" +wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" + +readonly WSREP_SST_OPT_CONF="$wsrep_defaults" + wsrep_defaults="$WSREP_SST_OPT_DEFAULT" -if [ -n "$wsrep_defaults" ]; then - wsrep_defaults="$wsrep_defaults " -fi -wsrep_defaults="$wsrep_defaults$WSREP_SST_OPT_EXTRA_DEFAULT" -if [ -n "$wsrep_defaults" ]; then - wsrep_defaults="$wsrep_defaults " -fi -readonly WSREP_SST_OPT_CONF="$wsrep_defaults$WSREP_SST_OPT_SUFFIX_DEFAULT" -readonly MY_PRINT_DEFAULTS="$MY_PRINT_DEFAULTS $WSREP_SST_OPT_CONF" +wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULT" +wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" + +readonly WSREP_SST_OPT_CONF_UNQUOTED="$wsrep_defaults" # # User can specify mariabackup specific settings that will be used during sst @@ -663,13 +697,21 @@ parse_cnf() # If the group name is the same as the "mysqld" without "--" prefix, # then try to use it together with the group suffix: if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then - reval=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern") + reval=$("$MY_PRINT_DEFAULTS" \ + ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \ + ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \ + ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \ + "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern") if [ -n "$reval" ]; then break fi fi # Let's try to use the group name as it is: - reval=$($MY_PRINT_DEFAULTS "$group" | awk "$pattern") + reval=$("$MY_PRINT_DEFAULTS" \ + ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \ + ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \ + ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \ + "$group" | awk "$pattern") if [ -n "$reval" ]; then break fi @@ -710,13 +752,21 @@ in_config() # If the group name is the same as the "mysqld" without "--" prefix, # then try to use it together with the group suffix: if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then - found=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern") + found=$("$MY_PRINT_DEFAULTS" \ + ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \ + ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \ + ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \ + "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern") if [ $found -ne 0 ]; then break fi fi # Let's try to use the group name as it is: - found=$($MY_PRINT_DEFAULTS "$group" | awk "$pattern") + found=$($MY_PRINT_DEFAULTS \ + ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \ + ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \ + ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \ + "$group" | awk "$pattern") if [ $found -ne 0 ]; then break fi @@ -797,29 +847,6 @@ else SST_PROGRESS_FILE="" fi -wsrep_log() -{ - # echo everything to stderr so that it gets into common error log - # deliberately made to look different from the rest of the log - local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)" - echo "WSREP_SST: $* ($tst)" >&2 -} - -wsrep_log_error() -{ - wsrep_log "[ERROR] $*" -} - -wsrep_log_warning() -{ - wsrep_log "[WARNING] $*" -} - -wsrep_log_info() -{ - wsrep_log "[INFO] $*" -} - wsrep_cleanup_progress_file() { [ -n "$SST_PROGRESS_FILE" -a \ @@ -959,6 +986,43 @@ check_sockets_utils() fi } +# +# Check if the port is in the "listen" state. +# The first parameter is the PID of the process that should +# listen on the port - if it is not known, you can specify +# an empty string or zero. +# The second parameter is the port number. +# The third parameter is a list of the names of utilities +# (via "|") that can listen on this port during the state +# transfer. +# +check_port() +{ + local pid="$1" + local port="$2" + local utils="$3" + + [ -z "$pid" ] || [ $pid -eq 0 ] && pid='[0-9]+' + + local rc=1 + + if [ $lsof_available -ne 0 ]; then + lsof -Pnl -i ":$port" 2>/dev/null | \ + grep -q -E "^($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*\\(LISTEN\\)" && rc=0 + elif [ $sockstat_available -ne 0 ]; then + sockstat -p "$port" 2>/dev/null | \ + grep -q -E "[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*[[:space:]]LISTEN" && rc=0 + elif [ $ss_available -ne 0 ]; then + ss -nlpH "( sport = :$port )" 2>/dev/null | \ + grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 + else + wsrep_log_error "unknown sockets utility" + exit 2 # ENOENT + fi + + return $rc +} + # # If the ssl_dhparams variable is already set, uses that as a source # of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in @@ -966,28 +1030,95 @@ check_sockets_utils() # check_for_dhparams() { - if [ -z "$ssl_dhparams" ]; then - ssl_dhparams="$DATA/dhparams.pem" - if [ ! -r "$ssl_dhparams" ]; then - get_openssl - if [ -n "$OPENSSL_BINARY" ]; then - wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams" - if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1 - then - wsrep_log_error "******** ERROR *****************************************" - wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *" - wsrep_log_error "********************************************************" - ssl_dhparams="" - fi - else - # Rollback: if openssl is not installed, then use - # the default parameters: + ssl_dhparams="$DATA/dhparams.pem" + if [ ! -r "$ssl_dhparams" ]; then + get_openssl + if [ -n "$OPENSSL_BINARY" ]; then + wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams" + if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1 + then + wsrep_log_error "******** ERROR *****************************************" + wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *" + wsrep_log_error "********************************************************" ssl_dhparams="" - fi + fi + else + # Rollback: if openssl is not installed, then use + # the default parameters: + ssl_dhparams="" fi fi } +# +# Verifies that the CA file verifies the certificate. +# Doing this here lets us generate better error messages. +# +# 1st param: path to the CA file. +# 2nd param: path to the certificate. +# +verify_ca_matches_cert() +{ + local ca_path="$1" + local cert_path="$2" + + # If the openssl utility is not installed, then + # we will not do this certificate check: + get_openssl + if [ -z "$OPENSSL_BINARY" ]; then + return + fi + + if ! "$OPENSSL_BINARY" verify -verbose -CAfile "$ca_path" "$cert_path" >/dev/null 2>&1 + then + wsrep_log_error "******** FATAL ERROR ********************************************" + wsrep_log_error "* The certifcate and CA (certificate authority) do not match. *" + wsrep_log_error "* It does not appear that the certificate was issued by the CA. *" + wsrep_log_error "* Please check your certificate and CA files. *" + wsrep_log_error "*****************************************************************" + exit 22 + fi +} + +# +# Verifies that the certificate matches the private key. +# Doing this will save us having to wait for a timeout that would +# otherwise occur. +# +# 1st param: path to the certificate. +# 2nd param: path to the private key. +# +verify_cert_matches_key() +{ + local cert_path="$1" + local key_path="$2" + + # If the diff utility is not installed, then + # we will not do this certificate check: + if [ -z "$(command -v diff)" ]; then + return + fi + + # If the openssl utility is not installed, then + # we will not do this certificate check: + get_openssl + if [ -z "$OPENSSL_BINARY" ]; then + return + fi + + # Generate the public key from the cert and the key. + # They should match (otherwise we can't create an SSL connection). + if ! diff <("$OPENSSL_BINARY" x509 -in "$cert_path" -pubkey -noout 2>/dev/null) \ + <("$OPENSSL_BINARY" pkey -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1 + then + wsrep_log_error "******************* FATAL ERROR ****************" + wsrep_log_error "* The certifcate and private key do not match. *" + wsrep_log_error "* Please check your certificate and key files. *" + wsrep_log_error "************************************************" + exit 22 + fi +} + # # Compares two version strings. # The first parameter is the version to be checked; @@ -996,22 +1127,22 @@ check_for_dhparams() # check_for_version() { - y1=${1#*.} + y1="${1#*.}" [ "$y1" = "$1" ] && y1="" z1=${y1#*.} [ "$z1" = "$y1" ] && z1="" - x1=${1%%.*} - y1=${y1%%.*} - z1=${z1%%.*} + x1="${1%%.*}" + y1="${y1%%.*}" + z1="${z1%%.*}" [ -z "$y1" ] && y1=0 [ -z "$z1" ] && z1=0 - y2=${2#*.} + y2="${2#*.}" [ "$y2" = "$2" ] && y2="" - z2=${y2#*.} + z2="${y2#*.}" [ "$z2" = "$y2" ] && z2="" - x2=${2%%.*} - y2=${y2%%.*} - z2=${z2%%.*} + x2="${2%%.*}" + y2="${y2%%.*}" + z2="${z2%%.*}" [ -z "$y2" ] && y2=0 [ -z "$z2" ] && z2=0 [ $x1 -lt $x2 ] && return 1 @@ -1032,8 +1163,8 @@ trim_string() if [ $x -ne $z ]; then local y="${1%$pattern*}" y=${#y} - x=$(( $z-$x-1 )) - y=$(( $y-$x+1 )) + x=$(( z-x-1 )) + y=$(( y-x+1 )) printf '%s' "${1:$x:$y}" else printf '' @@ -1043,3 +1174,105 @@ trim_string() echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" fi } + +# +# Check whether process is still running. +# The first parameter contains the name of the PID file. +# The second parameter is the flag of the need to delete +# the PID file. +# If the second parameter is not zero and not empty, +# then if the process terminates, the corresponding +# PID file will be deleted. +# This function also sets the CHECK_PID variable to zero +# if the process has already exited, or writes the PID +# of the process there if it is still running. +# +check_pid() +{ + local pid_file="$1" + local remove=${2:-0} + if [ -r "$pid_file" ]; then + local pid=$(cat "$pid_file" 2>/dev/null) + if [ -n "$pid" ]; then + if [ $pid -ne 0 ]; then + if ps -p "$pid" >/dev/null 2>&1; then + CHECK_PID=$pid + return 0 + fi + fi + fi + if [ $remove -eq 1 ]; then + rm -f "$pid_file" + fi + fi + CHECK_PID=0 + return 1 +} + +# +# Checking that the process with the specified PID is still +# running and killing it in this case by sending SIGTERM +# (using the "kill" operation). +# The first parameter contains PID of the process. +# The second and third parameters (both optional) are the names +# of the PID and the configuration files, which should be removed +# after the process ends. +# If the first parameter (PID of the process) is zero, then +# the function immediately deletes the PID and the configuration +# files (if specified), without any additional checks. +# +cleanup_pid() +{ + local pid="$1" + local pid_file="${2:-}" + local config="${3:-}" + + if [ $pid -ne 0 ]; then + if ps -p $pid >/dev/null 2>&1; then + if kill $pid >/dev/null 2>&1; then + sleep 0.5 + local round=0 + local force=0 + while ps -p $pid >/dev/null 2>&1; do + sleep 1 + round=$(( round+1 )) + if [ $round -eq 16 ]; then + if [ $force -eq 0 ]; then + round=8 + force=1 + kill -9 $pid >/dev/null 2>&1 + else + return 1; + fi + fi + done + elif ps -p $pid >/dev/null 2>&1; then + wsrep_log_warning "Unable to kill PID=$pid ($pid_file)" + return 1 + fi + fi + fi + + [ -n "$pid_file" ] && [ -f "$pid_file" ] && rm -f "$pid_file" + [ -n "$config" ] && [ -f "$config" ] && rm -f "$config" + + return 0 +} + +nproc="" + +get_proc() +{ + if [ -z "$nproc" ]; then + set +e + if [ "$OS" = 'Linux' ]; then + nproc=$(grep -c processor /proc/cpuinfo 2>/dev/null) + elif [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ]; then + nproc=$(sysctl -n hw.ncpu) + fi + if [ -z "$nproc" ] || [ $nproc -eq 0 ]; then + nproc=1 + fi + set -e + fi +} diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 0fdc765ff8b..7f97d9e8dea 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -29,11 +29,10 @@ eformat="" ekey="" ekeyfile="" encrypt=0 -nproc=1 ecode=0 ssyslog="" ssystag="" -MARIABACKUP_PID="" +BACKUP_PID="" tcert="" tpem="" tkey="" @@ -77,6 +76,11 @@ compress='none' compress_chunk="" compress_threads="" +backup_threads="" + +encrypt_threads="" +encrypt_chunk="" + readonly SECRET_TAG="secret" # Required for backup locks @@ -90,8 +94,8 @@ fi pcmd="pv $pvopts" declare -a RC -MARIABACKUP_BIN="$(command -v mariabackup)" -if [ ! -x "$MARIABACKUP_BIN" ]; then +BACKUP_BIN="$(command -v mariabackup)" +if [ ! -x "$BACKUP_BIN" ]; then wsrep_log_error 'mariabackup binary not found in path' exit 42 fi @@ -108,7 +112,8 @@ INNOBACKUPLOG="$DATA/mariabackup.backup.log" # Setting the path for ss and ip export PATH="/usr/sbin:/sbin:$PATH" -timeit(){ +timeit() +{ local stage="$1" shift local cmd="$@" @@ -198,6 +203,12 @@ get_keys() else ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'" fi + if [ -n "$encrypt_threads" ]; then + ecmd="$ecmd --encrypt-threads=$encrypt_threads" + fi + if [ -n "$encrypt_chunk" ]; then + ecmd="$ecmd --encrypt-chunk-size=$encrypt_chunk" + fi else wsrep_log_error "Unknown encryption format='$eformat'" exit 2 @@ -298,15 +309,6 @@ get_transfer() exit 2 fi - # Determine the socat version - SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') - if [ -z "$SOCAT_VERSION" ]; then - wsrep_log_error "******** FATAL ERROR ******************" - wsrep_log_error "* Cannot determine the socat version. *" - wsrep_log_error "***************************************" - exit 2 - fi - local action='Decrypting' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr" @@ -315,10 +317,22 @@ get_transfer() action='Encrypting' fi - if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then - # socat versions < 1.7.3 will have 512-bit dhparams (too small) - # so create 2048-bit dhparams and send that as a parameter: - check_for_dhparams + if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then + if [ -z "$ssl_dhparams" ]; then + # Determine the socat version + SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') + if [ -z "$SOCAT_VERSION" ]; then + wsrep_log_error "******** FATAL ERROR ******************" + wsrep_log_error "* Cannot determine the socat version. *" + wsrep_log_error "***************************************" + exit 2 + fi + if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then + # socat versions < 1.7.3 will have 512-bit dhparams (too small) + # so create 2048-bit dhparams and send that as a parameter: + check_for_dhparams + fi + fi if [ -n "$ssl_dhparams" ]; then tcmd="$tcmd,dhparam='$ssl_dhparams'" fi @@ -330,6 +344,11 @@ get_transfer() wsrep_log_error "Both PEM and CRT files required" exit 22 fi + if [ ! -r "$tpem" -o ! -r "$tcert" ]; then + wsrep_log_error "Both PEM and CRT files must be readable" + exit 22 + fi + verify_ca_matches_cert "$tcert" "$tpem" tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt" stagemsg="$stagemsg-OpenSSL-Encrypted-2" wsrep_log_info "$action with cert=$tpem, cafile=$tcert" @@ -339,6 +358,11 @@ get_transfer() wsrep_log_error "Both certificate and key files required" exit 22 fi + if [ ! -r "$tpem" -o ! -r "$tkey" ]; then + wsrep_log_error "Both certificate and key files must be readable" + exit 22 + fi + verify_cert_matches_key "$tpem" "$tkey" stagemsg="$stagemsg-OpenSSL-Encrypted-3" if [ -z "$tcert" ]; then if [ $encrypt -eq 4 ]; then @@ -350,6 +374,11 @@ get_transfer() wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0" else # CA verification + if [ ! -r "$tcert" ]; then + wsrep_log_error "Certificate file must be readable" + exit 22 + fi + verify_ca_matches_cert "$tcert" "$tpem" if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'" elif [ $encrypt -eq 4 ]; then @@ -440,10 +469,10 @@ read_cnf() tpem=$(parse_cnf 'sst' 'tcert') tkey=$(parse_cnf 'sst' 'tkey') fi - if [ "$tmode" != 'DISABLED' ] - then # backward-incompatible behavior - if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ] - then # no old-style SSL config in [sst] + if [ "$tmode" != 'DISABLED' ]; then + # backward-incompatible behavior + if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]; then + # no old-style SSL config in [sst] check_server_ssl_config fi if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ] @@ -504,21 +533,28 @@ read_cnf() compress_threads=$(parse_cnf "$encgroups" 'compress-threads') fi fi + + backup_threads=$(parse_cnf "$encgroups" 'backup-threads') + + if [ "$eformat" = 'xbcrypt' ]; then + encrypt_threads=$(parse_cnf "$encgroups" 'encrypt-threads') + encrypt_chunk=$(parse_cnf "$encgroups" 'encrypt-chunk-size') + fi } get_stream() { if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then sfmt='mbstream' - MBSTREAM_BIN="$(command -v mbstream)" - if [ -z "$MBSTREAM_BIN" ]; then + STREAM_BIN="$(command -v mbstream)" + if [ -z "$STREAM_BIN" ]; then wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path" exit 42 fi if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - strmcmd="'$MBSTREAM_BIN' -x" + strmcmd="'$STREAM_BIN' -x" else - strmcmd="'$MBSTREAM_BIN' -c '$INFO_FILE'" + strmcmd="'$STREAM_BIN' -c '$INFO_FILE'" fi else sfmt='tar' @@ -531,79 +567,33 @@ get_stream() wsrep_log_info "Streaming with $sfmt" } -get_proc() -{ - set +e - nproc=$(grep -c processor /proc/cpuinfo) - [ -z $nproc -o $nproc -eq 0 ] && nproc=1 - set -e -} - sig_joiner_cleanup() { wsrep_log_error "Removing $MAGIC_FILE file due to signal" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" } -cleanup_joiner() +cleanup_at_exit() { # Since this is invoked just after exit NNN local estatus=$? if [ $estatus -ne 0 ]; then wsrep_log_error "Cleanup after exit with status:$estatus" - elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + fi + + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file - fi - if [ -n "$progress" -a -p "$progress" ]; then - wsrep_log_info "Cleaning up fifo file $progress" - rm "$progress" - fi - - if [ -n "$STATDIR" ]; then - [ -d "$STATDIR" ] && rm -rf "$STATDIR" - fi - - # Final cleanup - pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') - - # This means no setsid done in mysqld. - # We don't want to kill mysqld here otherwise. - if [ $$ -eq $pgid ]; then - # This means a signal was delivered to the process. - # So, more cleanup. - if [ $estatus -ge 128 ]; then - kill -KILL -$$ || true + else + if [ -n "$BACKUP_PID" ]; then + if check_pid "$BACKUP_PID" 1; then + wsrep_log_error "mariabackup process is still running. Killing..." + cleanup_pid $CHECK_PID "$BACKUP_PID" + fi fi + [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" fi - exit $estatus -} - -check_pid() -{ - local pid_file="$1" - [ -r "$pid_file" ] && ps -p $(cat "$pid_file") 2>&1 >/dev/null -} - -cleanup_donor() -{ - # Since this is invoked just after exit NNN - local estatus=$? - if [ $estatus -ne 0 ]; then - wsrep_log_error "Cleanup after exit with status:$estatus" - fi - - if [ -n "$MARIABACKUP_PID" ]; then - if check_pid $MARIABACKUP_PID - then - wsrep_log_error "mariabackup process is still running. Killing..." - kill_mariabackup - fi - fi - - [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" - if [ -n "$progress" -a -p "$progress" ]; then wsrep_log_info "Cleaning up fifo file $progress" rm -f "$progress" || true @@ -611,8 +601,14 @@ cleanup_donor() wsrep_log_info "Cleaning up temporary directories" - [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true - [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + if [ -n "$STATDIR" ]; then + [ -d "$STATDIR" ] && rm -rf "$STATDIR" + fi + else + [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true + [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true + fi # Final cleanup pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') @@ -623,21 +619,13 @@ cleanup_donor() # This means a signal was delivered to the process. # So, more cleanup. if [ $estatus -ge 128 ]; then - kill -KILL -$$ || true + kill -KILL -- -$$ || true fi fi exit $estatus } -kill_mariabackup() -{ - local PID=$(cat "$MARIABACKUP_PID") - [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : - wsrep_log_info "Removing mariabackup pid file ($MARIABACKUP_PID)" - rm -f "$MARIABACKUP_PID" || true -} - setup_ports() { SST_PORT="$WSREP_SST_OPT_PORT" @@ -648,51 +636,17 @@ setup_ports() fi } -check_port() -{ - local PORT="$1" - local UTILS="$2" - - local port_info is_util - - if [ $lsof_available -ne 0 ]; then - port_info=$(lsof -i ":$PORT" -Pn 2>/dev/null | \ - grep -F '(LISTEN)') - is_util=$(echo "$port_info" | \ - grep -E "^($UTILS)[^[:space:]]*[[:space:]]+[0-9]+[[:space:]]+") - elif [ $sockstat_available -ne 0 ]; then - port_info=$(sockstat -p "$PORT" 2>/dev/null | \ - grep -F 'LISTEN') - is_util=$(echo "$port_info" | \ - grep -E "[[:space:]]+($UTILS)[^[:space:]]*[[:space:]]+[0-9]+[[:space:]]+") - elif [ $ss_available -ne 0 ]; then - port_info=$(ss -H -p -n -l "( sport = :$PORT )" 2>/dev/null) - is_util=$(echo "$port_info" | \ - grep -E "users:\\(.*\\(\"($UTILS)[^[:space:]]*\".*\.*\\)") - else - wsrep_log_error "unknown sockets utility" - exit 2 # ENOENT - fi - - if [ -z "$is_util" ]; then - return 1 - fi - - return 0 -} - -# waits ~10 seconds for nc to open the port and then reports ready -# (regardless of timeout) +# +# Waits ~30 seconds for socat or nc to open the port and +# then reports ready, regardless of timeout. +# wait_for_listen() { local PORT="$1" local ADDR="$2" local MODULE="$3" - - for i in {1..50} - do - if check_port "$PORT" 'socat|nc' - then + for i in {1..150}; do + if check_port "" "$PORT" 'socat|nc'; then break fi sleep 0.2 @@ -708,8 +662,8 @@ check_extra() if [ "$thread_handling" = 'pool-of-threads' ]; then local eport=$(parse_cnf '--mysqld' 'extra-port') if [ -n "$eport" ]; then - # mariabackup works only locally, hence, - # setting host to 127.0.0.1 unconditionally: + # mariabackup works only locally. + # Hence, setting host to 127.0.0.1 unconditionally: wsrep_log_info "SST through extra_port $eport" INNOEXTRA="$INNOEXTRA --host=127.0.0.1 --port=$eport" use_socket=0 @@ -825,11 +779,12 @@ monitor_process() local sst_stream_pid=$1 while true ; do - if ! ps -p "$WSREP_SST_OPT_PARENT" &>/dev/null; then + if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." + kill -- -"$WSREP_SST_OPT_PARENT" exit 32 fi - if ! ps -p "$sst_stream_pid" &>/dev/null; then + if ! ps -p "$sst_stream_pid" >/dev/null 2>&1; then break fi sleep 0.1 @@ -839,14 +794,14 @@ monitor_process() [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then - wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" + wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi read_cnf setup_ports -if "$MARIABACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then disver='--no-version-check' fi @@ -942,8 +897,8 @@ else gzip "$newfile" fi fi - INNOAPPLY="&> '$INNOAPPLYLOG'" - INNOMOVE="&> '$INNOMOVELOG'" + INNOAPPLY="> '$INNOAPPLYLOG' 2>&1" + INNOMOVE="> '$INNOMOVELOG' 2>&1" INNOBACKUP="2> '$INNOBACKUPLOG'" fi @@ -953,9 +908,9 @@ setup_commands() if [ -n "$WSREP_SST_OPT_MYSQLD" ]; then mysqld_args="--mysqld-args $WSREP_SST_OPT_MYSQLD" fi - INNOAPPLY="$MARIABACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY" - INNOMOVE="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" - INNOBACKUP="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP" + INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY" + INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" + INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP" } get_stream @@ -963,7 +918,7 @@ get_transfer if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] then - trap cleanup_donor EXIT + trap cleanup_at_exit EXIT if [ $WSREP_SST_OPT_BYPASS -eq 0 ] then @@ -976,12 +931,15 @@ then tmpdir=$(parse_cnf "$encgroups" 'tmpdir') if [ -z "$tmpdir" ]; then xtmpdir="$(mktemp -d)" - tmpopts="--tmpdir='$xtmpdir'" - wsrep_log_info "Using $xtmpdir as mariabackup temporary directory" + else + xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir") fi + wsrep_log_info "Using '$xtmpdir' as mariabackup temporary directory" + tmpopts="--tmpdir='$xtmpdir'" + itmpdir="$(mktemp -d)" - wsrep_log_info "Using $itmpdir as mariabackup temporary directory" + wsrep_log_info "Using '$itmpdir' as mariabackup working directory" usrst=0 if [ -n "$WSREP_SST_OPT_USER" ]; then @@ -1048,25 +1006,29 @@ then tcmd="$ecmd | $tcmd" fi - iopts="$iopts --databases-exclude='lost+found'" + iopts="--databases-exclude='lost+found' $iopts" if [ ${FORCE_FTWRL:-0} -eq 1 ]; then wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" - iopts="$iopts --no-backup-locks" + iopts="--no-backup-locks $iopts" fi # if compression is enabled for backup files, then add the # appropriate options to the mariabackup command line: if [ "$compress" != 'none' ]; then - iopts="$iopts --compress${compress:+=$compress}" + iopts="--compress${compress:+=$compress} $iopts" if [ -n "$compress_threads" ]; then - iopts="$iopts --compress-threads=$compress_threads" + iopts="--compress-threads=$compress_threads $iopts" fi if [ -n "$compress_chunk" ]; then - iopts="$iopts --compress-chunk-size=$compress_chunk" + iopts="--compress-chunk-size=$compress_chunk $iopts" fi fi + if [ -n "$backup_threads" ]; then + iopts="--parallel=$backup_threads $iopts" + fi + setup_commands set +e timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -1082,7 +1044,7 @@ then fi # mariabackup implicitly writes PID to fixed location in $xtmpdir - MARIABACKUP_PID="$xtmpdir/xtrabackup_pid" + BACKUP_PID="$xtmpdir/xtrabackup_pid" else # BYPASS FOR IST @@ -1134,6 +1096,10 @@ then ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$backup_threads" ]; then + impts="--parallel=$backup_threads $impts" + fi + stagemsg='Joiner-Recv' sencrypted=1 @@ -1173,7 +1139,7 @@ then fi trap sig_joiner_cleanup HUP PIPE INT TERM - trap cleanup_joiner EXIT + trap cleanup_at_exit EXIT if [ -n "$progress" ]; then adjust_progress @@ -1196,7 +1162,7 @@ then recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1 - if ! ps -p "$WSREP_SST_OPT_PARENT" &>/dev/null + if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1 then wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." exit 32 @@ -1316,7 +1282,7 @@ then fi - wsrep_log_info "Preparing the backup at ${DATA}" + wsrep_log_info "Preparing the backup at $DATA" setup_commands timeit "mariabackup prepare stage" "$INNOAPPLY" @@ -1327,26 +1293,26 @@ then MAGIC_FILE="$TDATA/$INFO_FILE" - wsrep_log_info "Moving the backup to ${TDATA}" + wsrep_log_info "Moving the backup to $TDATA" timeit "mariabackup move stage" "$INNOMOVE" if [ $? -eq 0 ]; then - wsrep_log_info "Move successful, removing ${DATA}" + wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" DATA="$TDATA" else - wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" + wsrep_log_error "Move failed, keeping '$DATA' for further diagnosis" wsrep_log_error "Check syslog or '$INNOMOVELOG' for details" exit 22 fi else - wsrep_log_info "${IST_FILE} received from donor: Running IST" + wsrep_log_info "'$IST_FILE' received from donor: Running IST" fi if [ ! -r "$MAGIC_FILE" ]; then - wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" + wsrep_log_error "SST magic file '$MAGIC_FILE' not found/readable" exit 2 fi diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 4aa3f8e63d8..798bee1ac10 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -103,7 +103,7 @@ then DROP PREPARE stmt;" fi -MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\ +MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF_UNQUOTED "\ "$AUTH -h$WSREP_SST_OPT_HOST_UNESCAPED "\ "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10" @@ -140,7 +140,7 @@ then fi # NOTE: we don't use --routines here because we're dumping mysql.proc table -MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF $AUTH -S$WSREP_SST_OPT_SOCKET \ +MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF_UNQUOTED $AUTH -S$WSREP_SST_OPT_SOCKET \ --add-drop-database --add-drop-table --skip-add-locks --create-options \ --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \ --skip-comments --flush-privileges --all-databases --events" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 0cc13ee4191..19a4d19fded 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -1,7 +1,7 @@ #!/bin/bash -ue -# Copyright (C) 2010-2014 Codership Oy # Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2010-2014 Codership Oy # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,9 +19,8 @@ # This is a reference script for rsync-based state snapshot tansfer -RSYNC_PID= # rsync pid file -RSYNC_CONF= # rsync configuration file -RSYNC_REAL_PID= # rsync process id +RSYNC_REAL_PID=0 # rsync process id +STUNNEL_REAL_PID=0 # stunnel process id OS="$(uname)" [ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH @@ -36,95 +35,95 @@ wsrep_check_programs rsync cleanup_joiner() { - wsrep_log_info "Joiner cleanup. rsync PID: $RSYNC_REAL_PID" - [ "0" != "$RSYNC_REAL_PID" ] && \ - kill $RSYNC_REAL_PID && \ - sleep 0.5 && \ - kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || : - [ -f "$RSYNC_CONF" ] && rm -f "$RSYNC_CONF" - [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" - [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$RSYNC_PID" ] && rm -f "$RSYNC_PID" + local failure=0 + + wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID, stunnel PID=$STUNNEL_REAL_PID" + + if [ -n "$STUNNEL" ]; then + if cleanup_pid $STUNNEL_REAL_PID "$STUNNEL_PID" "$STUNNEL_CONF"; then + if [ $RSYNC_REAL_PID -eq 0 ]; then + if [ -r "$RSYNC_PID" ]; then + RSYNC_REAL_PID=$(cat "$RSYNC_PID" 2>/dev/null) + if [ -z "$RSYNC_REAL_PID" ]; then + RSYNC_REAL_PID=0 + fi + fi + fi + else + wsrep_log_warning "stunnel cleanup failed." + failure=1 + fi + fi + + if [ $failure -eq 0 ]; then + if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then + [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + else + wsrep_log_warning "rsync cleanup failed." + fi + fi + wsrep_log_info "Joiner cleanup done." + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_cleanup_progress_file fi } -# Check whether rsync process is still running. -check_pid() -{ - local pid_file="$1" - [ -r "$pid_file" ] && ps -p $(cat "$pid_file") 2>&1 >/dev/null -} - check_pid_and_port() { local pid_file="$1" - local rsync_pid=$2 - local rsync_addr="$3" - local rsync_port="$4" + local pid=$2 + local addr="$3" + local port="$4" - if [ -z "$rsync_port" -o -z "$rsync_addr" -o -z "$rsync_pid" ]; then - wsrep_log_error "check_pid_and_port(): bad arguments" - exit 2 # ENOENT - fi + local utils='rsync|stunnel' - local port_info is_rsync + if ! check_port "$pid" "$port" "$utils"; then + local port_info + local busy=0 - if [ $lsof_available -ne 0 ]; then - port_info=$(lsof -i ":$rsync_port" -Pn 2>/dev/null | \ - grep -F '(LISTEN)') - is_rsync=$(echo "$port_info" | \ - grep -E "^(rsync|stunnel)[^[:space:]]*[[:space:]]+$rsync_pid[[:space:]]+") - elif [ $sockstat_available -ne 0 ]; then - port_info=$(sockstat -p "$rsync_port" 2>/dev/null | \ - grep -F 'LISTEN') - is_rsync=$(echo "$port_info" | \ - grep -E "[[:space:]]+(rsync|stunnel)[^[:space:]]*[[:space:]]+$rsync_pid[[:space:]]+") - elif [ $ss_available -ne 0 ]; then - port_info=$(ss -H -p -n -l "( sport = :$rsync_port )" 2>/dev/null) - is_rsync=$(echo "$port_info" | \ - grep -E "users:\\(.*\\(\"(rsync|stunnel)[^[:space:]]*\".*\.*\\)") - else - wsrep_log_error "unknown sockets utility" - exit 2 # ENOENT - fi - - if [ -z "$is_rsync" ]; then - local is_listening_all if [ $lsof_available -ne 0 ]; then - is_listening_all=$(echo "$port_info" | \ - grep -E "[[:space:]](\\*|\\[?::\\]?):$rsync_port[[:space:]]") + port_info=$(lsof -Pnl -i ":$port" 2>/dev/null | \ + grep -F '(LISTEN)') + echo "$port_info" | \ + grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port[[:space:]]" && busy=1 else - if [ $sockstat_available -eq 0 ]; then - port_info=$(echo "$port_info" | grep -q -F 'users:(') + local filter='([^[:space:]]+[[:space:]]+){4}[^[:space:]]+' + if [ $sockstat_available -eq 1 ]; then + port_info=$(sockstat -p "$port" 2>/dev/null | \ + grep -E '[[:space:]]LISTEN' | grep -o -E "$filter") + else + port_info=$(ss -nlpH "( sport = :$port )" 2>/dev/null | \ + grep -F 'users:(' | grep -o -E "$filter") fi - port_info=$(echo "$port_info" | \ - grep -E "[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+" -o) - is_listening_all=$(echo "$port_info" | \ - grep -E "[[:space:]](\\*|\\[?::\\]?):$rsync_port\$") + echo "$port_info" | \ + grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port\$" && busy=1 fi - local is_listening_addr=$(echo "$port_info" | \ - grep -w -F -- "$rsync_addr:$rsync_port") - if [ -z "$is_listening_addr" ]; then - is_listening_addr=$(echo "$port_info" | \ - grep -w -F "[$rsync_addr]:$rsync_port") + + if [ $busy -eq 0 ]; then + if echo "$port_info" | grep -qw -F "[$addr]:$port" || \ + echo "$port_info" | grep -qw -F -- "$addr:$port" + then + busy=1 + fi fi - if [ -n "$is_listening_all" -o -n "$is_listening_addr" ]; then - wsrep_log_error "rsync or stunnel daemon port '$rsync_port' " \ + + if [ $busy -eq 0 ]; then + return 1 + fi + + if ! check_port "$pid" "$port" "$utils"; then + wsrep_log_error "rsync or stunnel daemon port '$port' " \ "has been taken by another program" exit 16 # EBUSY fi - return 1 fi - check_pid "$pid_file" && [ $(cat "$pid_file") -eq $rsync_pid ] + check_pid "$pid_file" && [ $CHECK_PID -eq $pid ] } STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf" - STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" @@ -201,6 +200,8 @@ FILTER="-f '- /lost+found' -f '- /.zfs' -f '- /.fseventsd' -f '- /.Trashes' + -f '- /.pid' + -f '- /.conf' -f '+ /wsrep_sst_binlog.tar' -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' -f '- $INNODB_DATA_HOME_DIR/ibdata*' @@ -266,7 +267,7 @@ then else # check if the address is an ip-address (v4 or v6): if echo "$WSREP_SST_OPT_HOST_UNESCAPED" | \ - grep -q -E '^([0-9]+(\.[0-9]+){3,3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$' + grep -q -E '^([0-9]+(\.[0-9]+){3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$' then CHECK_OPT="checkIP = $WSREP_SST_OPT_HOST_UNESCAPED" else @@ -303,10 +304,10 @@ then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" + [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" if [ -n "$STUNNEL" ] then - [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" cat << EOF > "$STUNNEL_CONF" key = $SSTKEY cert = $SSTCERT @@ -321,6 +322,8 @@ ${VERIFY_OPT} ${CHECK_OPT} ${CHECK_OPT_LOCAL} EOF + else + [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" fi if [ $WSREP_SST_OPT_BYPASS -eq 0 ] @@ -329,13 +332,8 @@ EOF FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed" ERROR="$WSREP_SST_OPT_DATA/sst_error" - rm -rf "$FLUSHED" - rm -rf "$ERROR" - - # Use deltaxfer only for WAN - inv=$(basename "$0") - [ "$inv" = "wsrep_sst_rsync_wan" ] && WHOLE_FILE_OPT="" \ - || WHOLE_FILE_OPT="--whole-file" + [ -f "$FLUSHED" ] && rm -f "$FLUSHED" + [ -f "$ERROR" ] && rm -f "$ERROR" echo "flush tables" @@ -350,15 +348,14 @@ EOF if [ -f "$ERROR" ] then # Flush tables operation failed. - rm -rf "$ERROR" + rm -f "$ERROR" exit 255 fi - sleep 0.2 done STATE=$(cat "$FLUSHED") - rm -rf "$FLUSHED" + rm -f "$FLUSHED" sync @@ -385,6 +382,13 @@ EOF cd "$OLD_PWD" fi + # Use deltaxfer only for WAN + inv=$(basename "$0") + WHOLE_FILE_OPT="" + if [ "${inv%wsrep_sst_rsync_wan*}" != "$inv" ]; then + WHOLE_FILE_OPT="--whole-file" + fi + # first, the normal directories, so that we can detect incompatible protocol RC=0 eval rsync ${STUNNEL:+"'--rsh=$STUNNEL'"} \ @@ -436,16 +440,18 @@ EOF fi # then, we parallelize the transfer of database directories, - # use . so that path concatenation works: + # use '.' so that path concatenation works: cd "$WSREP_SST_OPT_DATA" - count=1 - [ "$OS" = 'Linux' ] && count=$(grep -c processor /proc/cpuinfo) - [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ] && count=$(sysctl -n hw.ncpu) + backup_threads=$(parse_cnf "--mysqld|sst" 'backup-threads') + if [ -z "$backup_threads" ]; then + get_proc + backup_threads=$nproc + fi find . -maxdepth 1 -mindepth 1 -type d -not -name 'lost+found' \ - -not -name '.zfs' -print0 | xargs -I{} -0 -P $count \ + -not -name '.zfs' -print0 | xargs -I{} -0 -P $backup_threads \ rsync ${STUNNEL:+--rsh="$STUNNEL"} \ --owner --group --perms --links --specials \ --ignore-times --inplace --recursive --delete --quiet \ @@ -484,35 +490,52 @@ EOF echo "done $STATE" + if [ -n "$STUNNEL" ]; then + [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" + [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" + fi + elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then check_sockets_utils - touch "$SST_PROGRESS_FILE" - MYSQLD_PID="$WSREP_SST_OPT_PARENT" - - MODULE="rsync_sst" - - RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" - # give some time for lingering rsync from previous SST to complete + # give some time for lingering stunnel from previous SST to complete check_round=0 - while check_pid "$RSYNC_PID" && [ $check_round -lt 10 ] + while check_pid "$STUNNEL_PID" 1 do - wsrep_log_info "lingering rsync daemon found at startup, waiting for it to exit" + wsrep_log_info "lingering stunnel daemon found at startup, waiting for it to exit" check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "stunnel daemon already running." + exit 114 # EALREADY + fi sleep 1 done - if check_pid "$RSYNC_PID" - then - wsrep_log_error "rsync daemon already running." - exit 114 # EALREADY - fi + MODULE="rsync_sst" + RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" + RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + + # give some time for lingering rsync from previous SST to complete + check_round=0 + while check_pid "$RSYNC_PID" 1 + do + wsrep_log_info "lingering rsync daemon found at startup, waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "rsync daemon already running." + exit 114 # EALREADY + fi + sleep 1 + done - [ -f "$RSYNC_PID" ] && rm -f "$RSYNC_PID" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" + if [ -z "$STUNNEL" ]; then + [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" + fi + ADDR="$WSREP_SST_OPT_ADDR" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST" @@ -522,7 +545,7 @@ then trap "exit 3" INT TERM ABRT trap cleanup_joiner EXIT - RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + touch "$SST_PROGRESS_FILE" if [ -n "${MYSQL_TMP_DIR:-}" ]; then SILENT="log file = $MYSQL_TMP_DIR/rsyncd.log" @@ -545,18 +568,18 @@ $SILENT path = $INNODB_DATA_HOME_DIR EOF -# rm -rf "$DATA"/ib_logfile* # we don't want old logs around +# rm -rf "$DATA/ib_logfile"* # we don't want old logs around - # If the IP is local listen only in it + # If the IP is local, listen only on it: if is_local_ip "$RSYNC_ADDR_UNESCAPED" then RSYNC_EXTRA_ARGS="--address $RSYNC_ADDR_UNESCAPED" STUNNEL_ACCEPT="$RSYNC_ADDR_UNESCAPED:$RSYNC_PORT" else - # Not local, possibly a NAT, listen on all interfaces + # Not local, possibly a NAT, listen on all interfaces: RSYNC_EXTRA_ARGS="" STUNNEL_ACCEPT="$RSYNC_PORT" - # Overwrite address with all + # Overwrite address with all: RSYNC_ADDR="*" fi @@ -564,8 +587,9 @@ EOF then rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS & RSYNC_REAL_PID=$! + TRANSFER_REAL_PID="$RSYNC_REAL_PID" + TRANSFER_PID=$RSYNC_PID else - [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" # Let's check if the path to the config file contains a space? if [ "${RSYNC_CONF#* }" = "$RSYNC_CONF" ]; then cat << EOF > "$STUNNEL_CONF" @@ -606,15 +630,11 @@ execargs = $SHELL -c \$RSYNC_CMD EOF fi stunnel "$STUNNEL_CONF" & - RSYNC_REAL_PID=$! - RSYNC_PID="$STUNNEL_PID" + STUNNEL_REAL_PID=$! + TRANSFER_REAL_PID="$STUNNEL_REAL_PID" + TRANSFER_PID=$STUNNEL_PID fi - until check_pid_and_port "$RSYNC_PID" "$RSYNC_REAL_PID" "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT" - do - sleep 0.2 - done - if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ] then # backward-incompatible behavior CN="" @@ -635,19 +655,26 @@ EOF ADDR="$CN:$MY_SECRET@$WSREP_SST_OPT_HOST" else MY_SECRET="" # for check down in recv_joiner() - ADDR=$WSREP_SST_OPT_HOST + ADDR="$WSREP_SST_OPT_HOST" fi + until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT" + do + sleep 0.2 + done + echo "ready $ADDR:$RSYNC_PORT/$MODULE" + MYSQLD_PID="$WSREP_SST_OPT_PARENT" + # wait for SST to complete by monitoring magic file - while [ ! -r "$MAGIC_FILE" ] && check_pid "$RSYNC_PID" && \ - ps -p $MYSQLD_PID >/dev/null + while [ ! -r "$MAGIC_FILE" ] && check_pid "$TRANSFER_PID" && \ + ps -p $MYSQLD_PID >/dev/null 2>&1 do sleep 1 done - if ! ps -p $MYSQLD_PID >/dev/null + if ! ps -p $MYSQLD_PID >/dev/null 2>&1 then wsrep_log_error \ "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly." @@ -698,7 +725,7 @@ EOF echo "rsync process ended without creating '$MAGIC_FILE'" fi - wsrep_cleanup_progress_file +# wsrep_cleanup_progress_file # cleanup_joiner else wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'" diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index b5ede2b29cb..24bff12219d 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -1,6 +1,6 @@ #!/bin/bash -ue -# Copyright (C) 2013 Percona Inc # Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,27 +23,31 @@ . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir +OS="$(uname)" ealgo="" +eformat="" ekey="" ekeyfile="" encrypt=0 -nproc=1 ecode=0 ssyslog="" ssystag="" -XTRABACKUP_PID="" +BACKUP_PID="" tcert="" tpem="" tkey="" +tmode="DISABLED" sockopt="" progress="" ttime=0 totime=0 +lsn="" ecmd="" rlimit="" # Initially -stagemsg="${WSREP_SST_OPT_ROLE}" +stagemsg="$WSREP_SST_OPT_ROLE" cpat="" +speciald=1 ib_home_dir="" ib_log_dir="" ib_undo_dir="" @@ -67,41 +71,54 @@ xtmpdir="" scomp="" sdecomp="" + ssl_dhparams="" -ssl_cert="" -ssl_ca="" -ssl_key="" +compress='none' +compress_chunk="" +compress_threads="" + +backup_threads="" + +encrypt_threads="" +encrypt_chunk="" + +readonly SECRET_TAG="secret" + +sst_ver=-1 if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then - pvopts+=$pvformat + pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" declare -a RC -INNOBACKUPEX_BIN='innobackupex' -DATA="${WSREP_SST_OPT_DATA}" +BACKUP_BIN="$(command -v innobackupex)" +if [ ! -x "$BACKUP_BIN" ]; then + wsrep_log_error 'innobackupex binary not found in path' + exit 42 +fi + +DATA="$WSREP_SST_OPT_DATA" INFO_FILE="xtrabackup_galera_info" IST_FILE="xtrabackup_ist" -MAGIC_FILE="${DATA}/${INFO_FILE}" +MAGIC_FILE="$DATA/$INFO_FILE" + +INNOAPPLYLOG="$DATA/innobackupex.prepare.log" +INNOMOVELOG="$DATA/innobackupex.move.log" +INNOBACKUPLOG="$DATA/innobackupex.backup.log" # Setting the path for ss and ip export PATH="/usr/sbin:/sbin:$PATH" -OS="$(uname)" - -if [ -z "$(command -v lsof)" ]; then - wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed." - exit 2 # ENOENT -fi - -timeit(){ +timeit() +{ local stage="$1" shift local cmd="$@" local x1 x2 took extcode - if [[ $ttime -eq 1 ]];then + if [ $ttime -eq 1 ]; then x1=$(date +%s) wsrep_log_info "Evaluating $cmd" eval "$cmd" @@ -121,105 +138,89 @@ timeit(){ get_keys() { # $encrypt -eq 1 is for internal purposes only - if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then + if [ $encrypt -ge 2 -o $encrypt -eq -1 ]; then return fi - if [[ $encrypt -eq 0 ]];then - if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then - wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + if [ $encrypt -eq 0 ]; then + if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then + wsrep_log_error "Options for encryption are specified, " \ + "but encryption itself is disabled. SST may fail." fi return fi - if [[ "$sfmt" == 'tar' ]];then - wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" + if [ $sfmt = 'tar' ]; then + wsrep_log_info "NOTE: key-based encryption (encrypt=1) " \ + "cannot be enabled with tar format" encrypt=-1 return fi - wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" + wsrep_log_info "Key based encryption enabled in my.cnf - supported only from Xtrabackup 2.1.4" - if [[ -z "$ealgo" ]];then + if [ -z "$ealgo" ]; then wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" exit 3 fi - if [[ -z "$ekey" && ! -r "$ekeyfile" ]];then + if [ -z "$ekey" -a ! -r "$ekeyfile" ]; then wsrep_log_error "FATAL: Either key or keyfile must be readable" exit 3 fi - if [[ -z "$ekey" ]];then - ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" - else - wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key" - wsrep_log_warning "to be set via the command-line and is considered insecure." - wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead." - - ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" - fi - - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - ecmd+=" -d" - fi - - stagemsg+="-XB-Encrypted" -} - -# -# verifies that the certificate matches the private key -# doing this will save us having to wait for a timeout that would -# otherwise occur. -# -# 1st param: path to the cert -# 2nd param: path to the private key -# -verify_cert_matches_key() -{ - local cert_path="$1" - local key_path="$2" - - wsrep_check_programs openssl diff - - # generate the public key from the cert and the key - # they should match (otherwise we can't create an SSL connection) - if ! diff <(openssl x509 -in "$cert_path" -pubkey -noout) <(openssl rsa -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1 - then - wsrep_log_error "******** FATAL ERROR ************************* " - wsrep_log_error "* The certifcate and private key do not match. " - wsrep_log_error "* Please check your certificate and key files. " - wsrep_log_error "********************************************** " - exit 22 - fi -} - -# Checks to see if the file exists -# If the file does not exist (or cannot be read), issues an error -# and exits -# -# 1st param: file name to be checked (for read access) -# 2nd param: 1st error message (header) -# 3rd param: 2nd error message (footer, optional) -# -verify_file_exists() -{ - local file_path="$1" - local error_message1="$2" - local error_message2="$3" - - if ! [[ -r "$file_path" ]]; then - wsrep_log_error "******** FATAL ERROR ************************* " - wsrep_log_error "* $error_message1 " - wsrep_log_error "* Could not find/access : $file_path " - - if ! [[ -z "$error_message2" ]]; then - wsrep_log_error "* $error_message2 " + if [ "$eformat" = 'openssl' ]; then + get_openssl + if [ -z "$OPENSSL_BINARY" ]; then + wsrep_log_error "If encryption using the openssl is enabled, " \ + "then you need to install openssl" + exit 2 fi - - wsrep_log_error "********************************************** " - exit 22 + ecmd="'$OPENSSL_BINARY' enc -$ealgo" + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + ecmd="$ecmd -pbkdf2" + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + ecmd="$ecmd -iter 1" + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + ecmd="$ecmd -md sha256" + fi + if [ -z "$ekey" ]; then + ecmd="$ecmd -kfile '$ekeyfile'" + else + ecmd="$ecmd -k '$ekey'" + fi + elif [ "$eformat" = 'xbcrypt' ]; then + if [ -z "$(command -v xbcrypt)" ]; then + wsrep_log_error "If encryption using the xbcrypt is enabled, " \ + "then you need to install xbcrypt" + exit 2 + fi + wsrep_log_info "NOTE: xbcrypt-based encryption, " \ + "supported only from Xtrabackup 2.1.4" + if [ -z "$ekey" ]; then + ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key-file='$ekeyfile'" + else + wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key" + wsrep_log_warning "to be set via the command-line and is considered insecure." + wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead." + ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'" + fi + if [ -n "$encrypt_threads" ]; then + ecmd="$ecmd --encrypt-threads=$encrypt_threads" + fi + if [ -n "$encrypt_chunk" ]; then + ecmd="$ecmd --encrypt-chunk-size=$encrypt_chunk" + fi + else + wsrep_log_error "Unknown encryption format='$eformat'" + exit 2 fi + + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + ecmd="$ecmd -d" + fi + + stagemsg="$stagemsg-XB-Encrypted" } get_transfer() @@ -310,15 +311,6 @@ get_transfer() exit 2 fi - # Determine the socat version - SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') - if [ -z "$SOCAT_VERSION" ]; then - wsrep_log_error "******** FATAL ERROR ******************" - wsrep_log_error "* Cannot determine the socat version. *" - wsrep_log_error "***************************************" - exit 2 - fi - local action='Decrypting' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr" @@ -327,11 +319,25 @@ get_transfer() action='Encrypting' fi - if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then - # socat versions < 1.7.3 will have 512-bit dhparams (too small) - # so create 2048-bit dhparams and send that as a parameter: - check_for_dhparams - tcmd="$tcmd,dhparam='$ssl_dhparams'" + if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then + if [ -z "$ssl_dhparams" ]; then + # Determine the socat version + SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') + if [ -z "$SOCAT_VERSION" ]; then + wsrep_log_error "******** FATAL ERROR ******************" + wsrep_log_error "* Cannot determine the socat version. *" + wsrep_log_error "***************************************" + exit 2 + fi + if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then + # socat versions < 1.7.3 will have 512-bit dhparams (too small) + # so create 2048-bit dhparams and send that as a parameter: + check_for_dhparams + fi + fi + if [ -n "$ssl_dhparams" ]; then + tcmd="$tcmd,dhparam='$ssl_dhparams'" + fi fi if [ $encrypt -eq 2 ]; then @@ -340,6 +346,11 @@ get_transfer() wsrep_log_error "Both PEM and CRT files required" exit 22 fi + if [ ! -r "$tpem" -o ! -r "$tcert" ]; then + wsrep_log_error "Both PEM and CRT files must be readable" + exit 22 + fi + verify_ca_matches_cert "$tcert" "$tpem" tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt" stagemsg="$stagemsg-OpenSSL-Encrypted-2" wsrep_log_info "$action with cert=$tpem, cafile=$tcert" @@ -349,6 +360,11 @@ get_transfer() wsrep_log_error "Both certificate and key files required" exit 22 fi + if [ ! -r "$tpem" -o ! -r "$tkey" ]; then + wsrep_log_error "Both certificate and key files must be readable" + exit 22 + fi + verify_cert_matches_key "$tpem" "$tkey" stagemsg="$stagemsg-OpenSSL-Encrypted-3" if [ -z "$tcert" ]; then if [ $encrypt -eq 4 ]; then @@ -360,6 +376,11 @@ get_transfer() wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0" else # CA verification + if [ ! -r "$tcert" ]; then + wsrep_log_error "Certificate file must be readable" + exit 22 + fi + verify_ca_matches_cert "$tcert" "$tpem" if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'" elif [ $encrypt -eq 4 ]; then @@ -387,13 +408,13 @@ get_footprint() { pushd "$WSREP_SST_OPT_DATA" 1>/dev/null payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }') - if [ $(in_config 'xtrabackup' 'compress') -eq 1 ]; then + if [ "$compress" != 'none' ]; then # QuickLZ has around 50% compression ratio # When compression/compaction used, the progress is only an approximate. payload=$(( payload*1/2 )) fi popd 1>/dev/null - pcmd+=" -s $payload" + pcmd="$pcmd -s $payload" adjust_progress } @@ -408,93 +429,141 @@ adjust_progress() return fi - if [[ -n "$progress" && "$progress" != '1' ]];then - if [[ -e "$progress" ]];then - pcmd+=" 2>>$progress" + if [ -n "$progress" -a "$progress" != '1' ]; then + if [ -e "$progress" ]; then + pcmd="$pcmd 2>>'$progress'" else - pcmd+=" 2>$progress" + pcmd="$pcmd 2>'$progress'" fi - elif [[ -z "$progress" && -n "$rlimit" ]];then + elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero pcmd="pv -q" fi - if [[ -n "$rlimit" && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then wsrep_log_info "Rate-limiting SST to $rlimit" - pcmd+=" -L \$rlimit" + pcmd="$pcmd -L \$rlimit" fi } +encgroups='--mysqld|sst|xtrabackup' + +check_server_ssl_config() +{ + tcert=$(parse_cnf "$encgroups" 'ssl-ca') + tpem=$(parse_cnf "$encgroups" 'ssl-cert') + tkey=$(parse_cnf "$encgroups" 'ssl-key') +} + read_cnf() { - sfmt=$(parse_cnf sst streamfmt "xbstream") - tfmt=$(parse_cnf sst transferfmt "socat") - tcert=$(parse_cnf sst tca "") - tpem=$(parse_cnf sst tcert "") - tkey=$(parse_cnf sst tkey "") - encrypt=$(parse_cnf sst encrypt 0) + sfmt=$(parse_cnf sst streamfmt 'xbstream') + tfmt=$(parse_cnf sst transferfmt 'socat') + + encrypt=$(parse_cnf "$encgroups" 'encrypt' 0) + tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | tr [:lower:] [:upper:]) + + if [ $encrypt -eq 0 -o $encrypt -ge 2 ] + then + if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ] + then + tcert=$(parse_cnf 'sst' 'tca') + tpem=$(parse_cnf 'sst' 'tcert') + tkey=$(parse_cnf 'sst' 'tkey') + fi + if [ "$tmode" != 'DISABLED' ]; then + # backward-incompatible behavior + if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]; then + # no old-style SSL config in [sst] + check_server_ssl_config + fi + if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ] + then + encrypt=3 # enable cert/key SSL encyption + + # avoid CA verification if not set explicitly: + # nodes may happen to have different CA if self-generated + # zeroing up tcert does the trick + [ "${tmode#VERIFY}" != "$tmode" ] || tcert="" + fi + fi + elif [ $encrypt -eq 1 ]; then + ealgo=$(parse_cnf "$encgroups" 'encrypt-algo') + eformat=$(parse_cnf "$encgroups" 'encrypt-format' 'xbcrypt') + ekey=$(parse_cnf "$encgroups" 'encrypt-key') + ekeyfile=$(parse_cnf "$encgroups" 'encrypt-key-file') + fi + + wsrep_log_info "SSL configuration: CA='$tcert', CERT='$tpem'," \ + "KEY='$tkey', MODE='$tmode', encrypt='$encrypt'" + sockopt=$(parse_cnf sst sockopt "") progress=$(parse_cnf sst progress "") rebuild=$(parse_cnf sst rebuild 0) ttime=$(parse_cnf sst time 0) cpat='.*\.pem$\|.*init\.ok$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$' [ "$OS" = 'FreeBSD' ] && cpat=$(echo "$cpat" | sed 's/\\|/|/g') - ealgo=$(parse_cnf xtrabackup encrypt "") - ekey=$(parse_cnf xtrabackup encrypt-key "") - ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + cpat=$(parse_cnf sst cpat "$cpat") scomp=$(parse_cnf sst compressor "") sdecomp=$(parse_cnf sst decompressor "") - # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html - if [[ -z "$ealgo" ]];then - ealgo=$(parse_cnf sst encrypt-algo "") - ekey=$(parse_cnf sst encrypt-key "") - ekeyfile=$(parse_cnf sst encrypt-key-file "") - fi - - # Pull the parameters needed for encrypt=4 - ssl_ca=$(parse_cnf sst ssl-ca "") - if [[ -z "$ssl_ca" ]]; then - ssl_ca=$(parse_cnf --mysqld ssl-ca "") - fi - ssl_cert=$(parse_cnf sst ssl-cert "") - if [[ -z "$ssl_cert" ]]; then - ssl_cert=$(parse_cnf --mysqld ssl-cert "") - fi - ssl_key=$(parse_cnf sst ssl-key "") - if [[ -z "$ssl_key" ]]; then - ssl_key=$(parse_cnf --mysqld ssl-key "") - fi - rlimit=$(parse_cnf sst rlimit "") uextra=$(parse_cnf sst use-extra 0) + speciald=$(parse_cnf sst sst-special-dirs 1) iopts=$(parse_cnf sst inno-backup-opts "") iapts=$(parse_cnf sst inno-apply-opts "") impts=$(parse_cnf sst inno-move-opts "") - stimeout=$(parse_cnf sst sst-initial-timeout 100) + stimeout=$(parse_cnf sst sst-initial-timeout 300) ssyslog=$(parse_cnf sst sst-syslog 0) ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}") - ssystag+="-" + ssystag="$ssystag-" + sstlogarchive=$(parse_cnf sst sst-log-archive 1) + sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir '/tmp/sst_log_archive') + + if [ $speciald -eq 0 ]; then + wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1" + speciald=1 + fi if [ $ssyslog -ne -1 ]; then ssyslog=$(in_config 'mysqld_safe' 'syslog') fi + + if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then + compress=$(parse_cnf "$encgroups" 'compress' 'none') + if [ "$compress" != 'none' ]; then + compress_chunk=$(parse_cnf "$encgroups" 'compress-chunk-size') + compress_threads=$(parse_cnf "$encgroups" 'compress-threads') + fi + fi + + backup_threads=$(parse_cnf "$encgroups" 'backup-threads') + + if [ "$eformat" = 'xbcrypt' ]; then + encrypt_threads=$(parse_cnf "$encgroups" 'encrypt-threads') + encrypt_chunk=$(parse_cnf "$encgroups" 'encrypt-chunk-size') + fi } get_stream() { if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then - XBSTREAM_BIN=$(command -v "$sfmt") - if [ -z "$XBSTREAM_BIN" ]; then - if [ -z "$XBSTREAM_BIN" ]; then - wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path" - exit 42 + STREAM_BIN=$(command -v "$sfmt") + if [ -z "$STREAM_BIN" ]; then + if [ "$sfmt" = 'xbstream' ]; then + STREAM_BIN="$(command -v mbstream)" + else + STREAM_BIN="$(command -v xbstream)" fi fi + if [ -z "$STREAM_BIN" ]; then + wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path" + exit 42 + fi if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - strmcmd="'$XBSTREAM_BIN' -x" + strmcmd="'$STREAM_BIN' -x" else - strmcmd="'$XBSTREAM_BIN' -c '$INFO_FILE'" + strmcmd="'$STREAM_BIN' -c '$INFO_FILE'" fi else sfmt='tar' @@ -507,91 +576,47 @@ get_stream() wsrep_log_info "Streaming with $sfmt" } -get_proc() -{ - set +e - nproc=$(grep -c processor /proc/cpuinfo) - [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 - set -e -} - sig_joiner_cleanup() { wsrep_log_error "Removing $MAGIC_FILE file due to signal" - rm -f "$MAGIC_FILE" + [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" } -cleanup_joiner() +cleanup_at_exit() { # Since this is invoked just after exit NNN local estatus=$? - if [[ $estatus -ne 0 ]];then + if [ $estatus -ne 0 ]; then wsrep_log_error "Cleanup after exit with status:$estatus" - elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + fi + + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file - fi - if [[ -n "$progress" && -p "$progress" ]];then - wsrep_log_info "Cleaning up fifo file $progress" - rm $progress - fi - if [[ -n "${STATDIR:-}" ]];then - [[ -d "$STATDIR" ]] && rm -rf "$STATDIR" - fi - - # Final cleanup - pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') - - # This means no setsid done in mysqld. - # We don't want to kill mysqld here otherwise. - if [[ $$ -eq $pgid ]];then - # This means a signal was delivered to the process. - # So, more cleanup. - if [[ $estatus -ge 128 ]];then - kill -KILL -$$ || true + else + if [ -n "$BACKUP_PID" ]; then + if check_pid "$BACKUP_PID" 1; then + wsrep_log_error "xtrabackup process is still running. Killing..." + cleanup_pid $CHECK_PID "$BACKUP_PID" + fi fi + [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" fi - exit $estatus -} - -check_pid() -{ - local pid_file="$1" - [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 -} - -cleanup_donor() -{ - # Since this is invoked just after exit NNN - local estatus=$? - if [[ $estatus -ne 0 ]];then - wsrep_log_error "Cleanup after exit with status:$estatus" - fi - - if [[ -n "${XTRABACKUP_PID:-}" ]];then - if check_pid "$XTRABACKUP_PID" - then - wsrep_log_error "xtrabackup process is still running. Killing... " - kill_xtrabackup - fi - fi - - rm -f "${DATA}/${IST_FILE}" || true - - if [[ -n "$progress" && -p "$progress" ]];then + if [ -n "$progress" -a -p "$progress" ]; then wsrep_log_info "Cleaning up fifo file $progress" rm -f "$progress" || true fi wsrep_log_info "Cleaning up temporary directories" - if [[ -n "$xtmpdir" ]];then - [[ -d "$xtmpdir" ]] && rm -rf "$xtmpdir" || true - fi - - if [[ -n "$itmpdir" ]];then - [[ -d "$itmpdir" ]] && rm -rf "$itmpdir" || true + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + if [ -n "$STATDIR" ]; then + [ -d "$STATDIR" ] && rm -rf "$STATDIR" + fi + else + [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true + [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true fi # Final cleanup @@ -599,44 +624,43 @@ cleanup_donor() # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. - if [[ $$ -eq $pgid ]];then - + if [ $$ -eq $pgid ]; then # This means a signal was delivered to the process. # So, more cleanup. - if [[ $estatus -ge 128 ]];then - kill -KILL -$$ || true + if [ $estatus -ge 128 ]; then + kill -KILL -- -$$ || true fi - fi exit $estatus } -kill_xtrabackup() +setup_ports() { - local PID=$(cat "$XTRABACKUP_PID") - [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : - wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID" - rm -f "$XTRABACKUP_PID" || true + SST_PORT="$WSREP_SST_OPT_PORT" + if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then + REMOTEIP="$WSREP_SST_OPT_HOST" + lsn="$WSREP_SST_OPT_LSN" + sst_ver="$WSREP_SST_OPT_SST_VER" + fi } -# waits ~1 minute for nc/socat to open the port and then reports ready -# (regardless of timeout) +# +# Waits ~30 seconds for socat or nc to open the port and +# then reports ready, regardless of timeout. +# wait_for_listen() { - local HOST="$1" - local PORT="$2" + local PORT="$1" + local ADDR="$2" local MODULE="$3" - local LSOF_OUT - - for i in {1..300} - do - LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c 2> /dev/null || :) - [ -n "${LSOF_OUT}" ] && break + for i in {1..150}; do + if check_port "" "$PORT" 'socat|nc'; then + break + fi sleep 0.2 done - - echo "ready ${HOST}:${PORT}/${MODULE}//${WSREP_SST_OPT_SST_VER:-1}" + echo "ready $ADDR/$MODULE//$sst_ver" } check_extra() @@ -648,9 +672,9 @@ check_extra() local eport=$(parse_cnf '--mysqld' 'extra-port') if [ -n "$eport" ]; then # Xtrabackup works only locally. - # Hence, setting host to 127.0.0.1 unconditionally. + # Hence, setting host to 127.0.0.1 unconditionally: wsrep_log_info "SST through extra_port $eport" - INNOEXTRA+=" --host=127.0.0.1 --port=$eport" + INNOEXTRA="$INNOEXTRA --host=127.0.0.1 --port=$eport" use_socket=0 else wsrep_log_error "Extra port $eport null, failing" @@ -661,7 +685,7 @@ check_extra() fi fi if [ $use_socket -eq 1 -a -n "$WSREP_SST_OPT_SOCKET" ]; then - INNOEXTRA+=" --socket='$WSREP_SST_OPT_SOCKET'" + INNOEXTRA="$INNOEXTRA --socket='$WSREP_SST_OPT_SOCKET'" fi } @@ -671,49 +695,71 @@ recv_joiner() local msg="$2" local tmt=$3 local checkf=$4 - local ltcmd + local wait=$5 - if [[ ! -d "${dir}" ]];then + if [ ! -d "$dir" ]; then # This indicates that IST is in progress return fi - pushd "${dir}" 1>/dev/null + local ltcmd="$tcmd" + if [ $tmt -gt 0 ]; then + if [ -n "$(command -v timeout)" ]; then + if timeout --help | grep -qw -- '-k'; then + ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" + else + ltcmd="timeout -s9 $tmt $tcmd" + fi + fi + fi + + pushd "$dir" 1>/dev/null set +e - if [ $tmt -gt 0 -a -n "$(command -v timeout)" ]; then - if timeout --help | grep -qw -- '-k';then - ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" - else - ltcmd="timeout -s9 $tmt $tcmd" - fi - timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" - else - timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + if [ $wait -ne 0 ]; then + wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" & fi + timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e popd 1>/dev/null - if [[ ${RC[0]} -eq 124 ]];then - wsrep_log_error "Possible timeout in receving first data from donor in gtid stage" + if [ ${RC[0]} -eq 124 ]; then + wsrep_log_error "Possible timeout in receiving first data from " \ + "donor in gtid stage: exit codes: ${RC[@]}" exit 32 fi - for ecode in "${RC[@]}";do - if [[ $ecode -ne 0 ]];then + for ecode in "${RC[@]}"; do + if [ $ecode -ne 0 ]; then wsrep_log_error "Error while getting data from donor node: " \ "exit codes: ${RC[@]}" exit 32 fi done - if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then - # this message should cause joiner to abort - wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" - wsrep_log_info "Contents of datadir" - wsrep_log_info $(ls -l "$dir/"*) - exit 32 + if [ $checkf -eq 1 ]; then + if [ ! -r "$MAGIC_FILE" ]; then + # this message should cause joiner to abort + wsrep_log_error "receiving process ended without creating " \ + "'$MAGIC_FILE'" + wsrep_log_info "Contents of datadir" + wsrep_log_info $(ls -l "$dir/"*) + exit 32 + fi + + # check donor supplied secret + SECRET=$(grep -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2) + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'" + exit 32 + fi + + # remove secret from magic file + grep -v -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new" + mv "$MAGIC_FILE.new" "$MAGIC_FILE" fi } @@ -722,14 +768,14 @@ send_donor() local dir="$1" local msg="$2" - pushd "${dir}" 1>/dev/null + pushd "$dir" 1>/dev/null set +e timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" set -e popd 1>/dev/null - for ecode in "${RC[@]}";do - if [[ $ecode -ne 0 ]];then + for ecode in "${RC[@]}"; do + if [ $ecode -ne 0 ]; then wsrep_log_error "Error while sending data to joiner node: " \ "exit codes: ${RC[@]}" exit 32 @@ -742,94 +788,47 @@ monitor_process() local sst_stream_pid=$1 while true ; do - - if ! ps --pid "${WSREP_SST_OPT_PARENT}" &>/dev/null; then + if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." - kill -- -"${WSREP_SST_OPT_PARENT}" + kill -- -"$WSREP_SST_OPT_PARENT" exit 32 fi - - if ! ps --pid "${sst_stream_pid}" &>/dev/null; then + if ! ps -p "$sst_stream_pid" >/dev/null 2>&1; then break fi - sleep 0.1 - done } -innobackup=$(command -v "$INNOBACKUPEX_BIN") -if [ ! -x "$innobackup" ]; then - wsrep_log_error "innobackupex not in path: $PATH" - exit 2 -fi - # check the version, we require XB-2.4 to ensure that we can pass the # datadir via the command-line option XB_REQUIRED_VERSION="2.3.5" -XB_VERSION=`$INNOBACKUPEX_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` +XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` if [[ -z "$XB_VERSION" ]]; then - wsrep_log_error "FATAL: Cannot determine the $INNOBACKUPEX_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" + wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" exit 2 fi if ! check_for_version "$XB_VERSION" "$XB_REQUIRED_VERSION"; then - wsrep_log_error "FATAL: The $INNOBACKUPEX_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" + wsrep_log_error "FATAL: The $BACKUP_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" exit 2 fi -rm -f "${MAGIC_FILE}" +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [[ ! "${WSREP_SST_OPT_ROLE}" == 'joiner' && ! "${WSREP_SST_OPT_ROLE}" == 'donor' ]];then - wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then + wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi read_cnf +setup_ports -if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then disver='--no-version-check' fi -iopts+=" --databases-exclude='lost+found'" - -if [ ${FORCE_FTWRL:-0} -eq 1 ]; then - wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" - iopts+=' --no-backup-locks' -fi - -if [[ $ssyslog -eq 1 ]];then - if [ -z "$(command -v logger)" ]; then - wsrep_log_error "logger not in path: $PATH. Ignoring" - else - wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog" - - exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) - - wsrep_log_error() - { - logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" - } - - wsrep_log_info() - { - logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" - } - - INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply" - INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move" - INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" - fi -else - INNOAPPLY="&>'$DATA/innobackup.prepare.log'" - INNOMOVE="&>'$DATA/innobackup.move.log'" - INNOBACKUP="2>'$DATA/innobackup.backup.log'" -fi - -get_stream -get_transfer - # if no command line argument and INNODB_DATA_HOME_DIR environment variable # is not set, try to get it from my.cnf: if [ -z "$INNODB_DATA_HOME_DIR" ]; then @@ -848,37 +847,123 @@ INNODB_DATA_HOME_DIR=$(pwd -P) cd "$OLD_PWD" -setup_commands () { - INNOAPPLY="$INNOBACKUPEX_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY" - INNOMOVE="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $impts --move-back --force-non-empty-directories '$DATA' $INNOMOVE" +if [ $ssyslog -eq 1 ]; then + if [ -n "$(command -v logger)" ]; then + wsrep_log_info "Logging all stderr of SST/xtrabackup to syslog" + + exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) + + wsrep_log_error() + { + logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" + } + + wsrep_log_info() + { + logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" + } + else + wsrep_log_error "logger not in path: $PATH. Ignoring" + fi + INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply" + INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move" + INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" +else + if [ $sstlogarchive -eq 1 ] + then + ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + + if [ -n "$sstlogarchivedir" ]; then + if [ ! -d "$sstlogarchivedir" ]; then + mkdir -p "$sstlogarchivedir" + fi + fi + + if [ -e "$INNOAPPLYLOG" ] + then + if [ -n "$sstlogarchivedir" ] + then + newfile=$(basename "$INNOAPPLYLOG") + newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" + else + newfile="$INNOAPPLYLOG.$ARCHIVETIMESTAMP" + fi + wsrep_log_info "Moving '$INNOAPPLYLOG' to '$newfile'" + mv "$INNOAPPLYLOG" "$newfile" + gzip "$newfile" + fi + + if [ -e "$INNOMOVELOG" ] + then + if [ -n "$sstlogarchivedir" ] + then + newfile=$(basename "$INNOMOVELOG") + newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" + else + newfile="$INNOMOVELOG.$ARCHIVETIMESTAMP" + fi + wsrep_log_info "Moving '$INNOMOVELOG' to '$newfile'" + mv "$INNOMOVELOG" "$newfile" + gzip "$newfile" + fi + + if [ -e "$INNOBACKUPLOG" ] + then + if [ -n "$sstlogarchivedir" ] + then + newfile=$(basename "$INNOBACKUPLOG") + newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" + else + newfile="$INNOBACKUPLOG.$ARCHIVETIMESTAMP" + fi + wsrep_log_info "Moving '$INNOBACKUPLOG' to '$newfile'" + mv "$INNOBACKUPLOG" "$newfile" + gzip "$newfile" + fi + fi + INNOAPPLY="> '$INNOAPPLYLOG' 2>&1" + INNOMOVE="> '$INNOMOVELOG' 2>&1" + INNOBACKUP="2> '$INNOBACKUPLOG'" +fi + +setup_commands() +{ + INNOAPPLY="$BACKUP_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY" + INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories '$DATA' $INNOMOVE" sfmt_work="$sfmt" if [ "$sfmt" = 'mbstream' ]; then sfmt_work='xbstream' fi - INNOBACKUP="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP" + INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP" } -if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +get_stream +get_transfer + +if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] then - trap cleanup_donor EXIT + trap cleanup_at_exit EXIT if [ $WSREP_SST_OPT_BYPASS -eq 0 ] then - if [ -z "$WSREP_SST_OPT_SST_VER" ]; then + if [ -z "$sst_ver" ]; then wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support" wsrep_log_error "The joiner is not supported for this version of donor" exit 93 fi - tmpdir=$(parse_cnf "--mysqld|sst|xtrabackup" 'tmpdir') + tmpdir=$(parse_cnf "$encgroups" 'tmpdir') if [ -z "$tmpdir" ]; then - xtmpdir=$(mktemp -d) - tmpopts=" --tmpdir='$xtmpdir'" - wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory" + xtmpdir="$(mktemp -d)" + else + xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir") fi - itmpdir=$(mktemp -d) - wsrep_log_info "Using $itmpdir as innobackupex temporary directory" + wsrep_log_info "Using '$xtmpdir' as xtrabackup temporary directory" + tmpopts="--tmpdir='$xtmpdir'" + + itmpdir="$(mktemp -d)" + wsrep_log_info "Using '$itmpdir' as xtrabackup working directory" usrst=0 if [ -n "$WSREP_SST_OPT_USER" ]; then @@ -887,37 +972,41 @@ then fi if [ -n "$WSREP_SST_OPT_PSWD" ]; then - export MYSQL_PWD="$WSREP_SST_OPT_PSWD" + INNOEXTRA="$INNOEXTRA --password='$WSREP_SST_OPT_PSWD'" elif [ $usrst -eq 1 ]; then # Empty password, used for testing, debugging etc. - unset MYSQL_PWD + INNOEXTRA="$INNOEXTRA --password=" fi - get_keys check_extra wsrep_log_info "Streaming GTID file before SST" # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id # (separated by a space). - echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" + echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE" + + if [ -n "$WSREP_SST_OPT_REMOTE_PSWD" ]; then + # Let joiner know that we know its secret + echo "$SECRET_TAG $WSREP_SST_OPT_REMOTE_PSWD" >> "$MAGIC_FILE" + fi ttcmd="$tcmd" - if [ $encrypt -eq 1 ]; then - if [ -n "$scomp" ]; then - tcmd="\$ecmd | $scomp | $tcmd" - else - tcmd="\$ecmd | $tcmd" - fi - elif [ -n "$scomp" ]; then + if [ -n "$scomp" ]; then tcmd="$scomp | $tcmd" fi - send_donor $DATA "${stagemsg}-gtid" + get_keys + if [ $encrypt -eq 1 ]; then + tcmd="$ecmd | $tcmd" + fi + + send_donor "$DATA" "$stagemsg-gtid" # Restore the transport commmand to its original state tcmd="$ttcmd" + if [ -n "$progress" ]; then get_footprint tcmd="$pcmd | $tcmd" @@ -929,34 +1018,57 @@ then wsrep_log_info "Sleeping before data transfer for SST" sleep 10 - wsrep_log_info "Streaming the backup to joiner at ${WSREP_SST_OPT_HOST}:${WSREP_SST_OPT_PORT}" + wsrep_log_info "Streaming the backup to joiner at $REMOTEIP:$SST_PORT" # Add compression to the head of the stream (if specified) - if [[ -n "$scomp" ]]; then + if [ -n "$scomp" ]; then tcmd="$scomp | $tcmd" fi # Add encryption to the head of the stream (if specified) - if [[ $encrypt -eq 1 ]]; then - tcmd="\$ecmd | $tcmd" + if [ $encrypt -eq 1 ]; then + tcmd="$ecmd | $tcmd" + fi + + iopts="--databases-exclude='lost+found' $iopts" + + if [ ${FORCE_FTWRL:-0} -eq 1 ]; then + wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" + iopts="--no-backup-locks $iopts" + fi + + # if compression is enabled for backup files, then add the + # appropriate options to the innobackupex command line: + if [ "$compress" != 'none' ]; then + iopts="--compress${compress:+=$compress} $iopts" + if [ -n "$compress_threads" ]; then + iopts="--compress-threads=$compress_threads $iopts" + fi + if [ -n "$compress_chunk" ]; then + iopts="--compress-chunk-size=$compress_chunk $iopts" + fi + fi + + if [ -n "$backup_threads" ]; then + iopts="--parallel=$backup_threads $iopts" fi setup_commands set +e - timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" set -e if [ ${RC[0]} -ne 0 ]; then - wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ - "Check ${DATA}/innobackup.backup.log" - exit 22 - elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then - wsrep_log_error "$tcmd finished with error: ${RC[1]}" - exit 22 + wsrep_log_error "innobackupex finished with error: ${RC[0]}. " \ + "Check syslog or '$INNOBACKUPLOG' for details" + exit 22 + elif [ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]; then + wsrep_log_error "$tcmd finished with error: ${RC[1]}" + exit 22 fi # innobackupex implicitly writes PID to fixed location in $xtmpdir - XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" + BACKUP_PID="$xtmpdir/xtrabackup_pid" else # BYPASS FOR IST @@ -965,31 +1077,31 @@ then # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id # (separated by a space). - echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" - echo "1" > "${DATA}/${IST_FILE}" - get_keys - if [ $encrypt -eq 1 ]; then - if [ -n "$scomp" ]; then - tcmd="\$ecmd | $scomp | $tcmd" - else - tcmd="\$ecmd | $tcmd" - fi - elif [ -n "$scomp" ]; then + echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE" + echo "1" > "$DATA/$IST_FILE" + + if [ -n "$scomp" ]; then tcmd="$scomp | $tcmd" fi - strmcmd+=" \${IST_FILE}" - send_donor "$DATA" "${stagemsg}-IST" + get_keys + if [ $encrypt -eq 1 ]; then + tcmd="$ecmd | $tcmd" + fi + + strmcmd="$strmcmd '$IST_FILE'" + + send_donor "$DATA" "$stagemsg-IST" fi - echo "done ${WSREP_SST_OPT_GTID}" + echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" -elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then - [[ -e "$SST_PROGRESS_FILE" ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" - [[ -n "$SST_PROGRESS_FILE" ]] && touch "$SST_PROGRESS_FILE" + [ -e "$SST_PROGRESS_FILE" ] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" ib_home_dir="$INNODB_DATA_HOME_DIR" @@ -1008,66 +1120,101 @@ then ib_undo_dir="$INNODB_UNDO_DIR" - stagemsg="Joiner-Recv" + if [ -n "$backup_threads" ]; then + impts="--parallel=$backup_threads $impts" + fi + + stagemsg='Joiner-Recv' sencrypted=1 nthreads=1 MODULE="xtrabackup_sst" - rm -f "${DATA}/${IST_FILE}" + [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" # May need xtrabackup_checkpoints later on - rm -f "${DATA}/xtrabackup_binary" "${DATA}/xtrabackup_galera_info" "${DATA}/xtrabackup_logfile" + [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" + [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" + [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0" - wait_for_listen ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT} ${MODULE} & + ADDR="$WSREP_SST_OPT_ADDR" + + if [ "${tmode#VERIFY}" != "$tmode" ] + then # backward-incompatible behavior + CN="" + if [ -n "$tpem" ] + then + # find out my Common Name + get_openssl + if [ -z "$OPENSSL_BINARY" ]; then + wsrep_log_error 'openssl not found but it is required for authentication' + exit 42 + fi + CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ + tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + sed s/\ %//) + fi + MY_SECRET="$(wsrep_gen_secret)" + # Add authentication data to address + ADDR="$CN:$MY_SECRET@$ADDR" + else + MY_SECRET="" # for check down in recv_joiner() + fi trap sig_joiner_cleanup HUP PIPE INT TERM - trap cleanup_joiner EXIT + trap cleanup_at_exit EXIT - if [[ -n "$progress" ]];then + if [ -n "$progress" ]; then adjust_progress - tcmd+=" | $pcmd" + tcmd="$tcmd | $pcmd" fi get_keys if [ $encrypt -eq 1 -a $sencrypted -eq 1 ]; then - if [ -n "$sdecomp" ]; then - strmcmd="$sdecomp | \$ecmd | $strmcmd" - else - strmcmd="\$ecmd | $strmcmd" - fi - elif [ -n "$sdecomp" ]; then - strmcmd="$sdecomp | $strmcmd" + strmcmd="$ecmd | $strmcmd" fi - STATDIR=$(mktemp -d) - MAGIC_FILE="${STATDIR}/${INFO_FILE}" - recv_joiner "$STATDIR" "${stagemsg}-gtid" $stimeout 1 + if [ -n "$sdecomp" ]; then + strmcmd="$sdecomp | $strmcmd" + fi - if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null + check_sockets_utils + + STATDIR="$(mktemp -d)" + MAGIC_FILE="$STATDIR/$INFO_FILE" + + recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1 + + if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1 then wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." exit 32 fi - if [ ! -r "${STATDIR}/${IST_FILE}" ] + if [ ! -r "$STATDIR/$IST_FILE" ] then - if [[ -d "${DATA}/.sst" ]];then - wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing" - rm -rf "${DATA}/.sst" + if [ -d "$DATA/.sst" ]; then + wsrep_log_info "WARNING: Stale temporary SST directory: '$DATA/.sst' from previous state transfer. Removing" + rm -rf "$DATA/.sst" fi - mkdir -p "${DATA}/.sst" - (recv_joiner "$DATA/.sst" "${stagemsg}-SST" 0 0) & + mkdir -p "$DATA/.sst" + (recv_joiner "$DATA/.sst" "$stagemsg-SST" 0 0 0) & jpid=$! wsrep_log_info "Proceeding with SST" wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" - if [ "${OS}" = "FreeBSD" ]; then - find -E $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ + if [ "$OS" = 'FreeBSD' ]; then + find -E ${ib_home_dir:+"$ib_home_dir"} \ + ${ib_undo_dir:+"$ib_undo_dir"} \ + ${ib_log_dir:+"$ib_log_dir"} \ + "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+ else - find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ + find ${ib_home_dir:+"$ib_home_dir"} \ + ${ib_undo_dir:+"$ib_undo_dir"} \ + ${ib_log_dir:+"$ib_log_dir"} \ + "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+ fi get_binlog @@ -1082,50 +1229,46 @@ then cd "$OLD_PWD" fi - TDATA="${DATA}" - DATA="${DATA}/.sst" + TDATA="$DATA" + DATA="$DATA/.sst" - MAGIC_FILE="${DATA}/${INFO_FILE}" + MAGIC_FILE="$DATA/$INFO_FILE" wsrep_log_info "Waiting for SST streaming to complete!" monitor_process $jpid - get_proc - - if [[ ! -s "${DATA}/xtrabackup_checkpoints" ]];then - wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor" + if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then + wsrep_log_error "xtrabackup_checkpoints missing, failed xtrabackup/SST on donor" exit 2 fi # Rebuild indexes for compact backups - if grep -q 'compact = 1' "${DATA}/xtrabackup_checkpoints";then + if grep -q -F 'compact = 1' "$DATA/xtrabackup_checkpoints"; then wsrep_log_info "Index compaction detected" - rebuild=1 - fi - - if [[ $rebuild -eq 1 ]];then - nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) + get_proc + nthreads=$(parse_cnf "$encgroups" 'rebuild-threads' $nproc) wsrep_log_info "Rebuilding during prepare with $nthreads threads" - rebuildcmd="--rebuild-indexes --rebuild-threads='$nthreads'" + rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" fi - qpfiles=$(find "${DATA}" -maxdepth 1 -type f -name '*.qp' -print -quit) + qpfiles=$(find "$DATA" -maxdepth 1 -type f -name '*.qp' -print -quit) if [ -n "$qpfiles" ]; then wsrep_log_info "Compressed qpress files found" if [ -z "$(command -v qpress)" ]; then - wsrep_log_error "qpress not found in path: $PATH" + wsrep_log_error "qpress utility not found in the path" exit 22 fi + get_proc + dcmd="xargs -n 2 qpress -dT$nproc" - if [[ -n "$progress" ]] && pv --help | grep -qw '--line-mode';then - count=$(find "${DATA}" -type f -name '*.qp' | wc -l) + if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) - if pv --help | grep -qw -F '-F';then - pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" - else - pvopts="-f -s $count -l -N Decompression" + pvopts="-f -s $count -l -N Decompression" + if pv --help | grep -qw -- '-F'; then + pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" adjust_progress @@ -1134,13 +1277,13 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit "Joiner-Decompression" "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" extcode=$? - if [[ $extcode -eq 0 ]];then + if [ $extcode -eq 0 ]; then wsrep_log_info "Removing qpress files after decompression" find "$DATA" -type f -name '*.qp' -delete - if [[ $? -ne 0 ]];then + if [ $? -ne 0 ]; then wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" fi else @@ -1165,38 +1308,39 @@ then fi - wsrep_log_info "Preparing the backup at ${DATA}" + wsrep_log_info "Preparing the backup at $DATA" setup_commands timeit "Xtrabackup prepare stage" "$INNOAPPLY" - if [ $? -ne 0 ]; - then - wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" + if [ $? -ne 0 ]; then + wsrep_log_error "xtrabackup apply finished with errors. Check '$INNOAPPLYLOG' for details" exit 22 fi - MAGIC_FILE="${TDATA}/${INFO_FILE}" - set +e - rm "$TDATA/innobackup.prepare.log" "$TDATA/innobackup.move.log" - set -e - wsrep_log_info "Moving the backup to ${TDATA}" + # [ -f "$INNOAPPLYLOG" ] && rm "$INNOAPPLYLOG" + + MAGIC_FILE="$TDATA/$INFO_FILE" + + wsrep_log_info "Moving the backup to $TDATA" timeit "Xtrabackup move stage" "$INNOMOVE" - if [[ $? -eq 0 ]];then - wsrep_log_info "Move successful, removing ${DATA}" + if [ $? -eq 0 ]; then + wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" - DATA="${TDATA}" + DATA="$TDATA" else - wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" - wsrep_log_error "Check ${DATA}/innobackup.move.log for details" + wsrep_log_error "Move failed, keeping '$DATA' for further diagnosis" + wsrep_log_error "Check syslog or '$INNOMOVELOG' for details" exit 22 fi else - wsrep_log_info "${IST_FILE} received from donor: Running IST" + + wsrep_log_info "'$IST_FILE' received from donor: Running IST" + fi - if [[ ! -r "${MAGIC_FILE}" ]];then - wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" + if [ ! -r "$MAGIC_FILE" ]; then + wsrep_log_error "SST magic file '$MAGIC_FILE' not found/readable" exit 2 fi diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh index c6dad8d1791..b84aca4866e 100644 --- a/scripts/wsrep_sst_xtrabackup.sh +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -97,8 +97,9 @@ get_keys() fi if [[ $encrypt -eq 0 ]];then - if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then - wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then + wsrep_log_error "Options for encryption are specified, " \ + "but encryption itself is disabled. SST may fail." fi return fi @@ -324,12 +325,6 @@ cleanup_joiner() fi } -check_pid() -{ - local pid_file="$1" - [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 -} - cleanup_donor() { # Since this is invoked just after exit NNN @@ -339,12 +334,11 @@ cleanup_donor() fi if [[ -n "$XTRABACKUP_PID" ]];then - if check_pid "$XTRABACKUP_PID" + if check_pid "$XTRABACKUP_PID" 1 then wsrep_log_error "xtrabackup process is still running. Killing... " - kill_xtrabackup + cleanup_pid $CHECK_PID "$XTRABACKUP_PID" fi - rm -f "$XTRABACKUP_PID" fi rm -f "${DATA}/${IST_FILE}" @@ -355,13 +349,6 @@ cleanup_donor() fi } -kill_xtrabackup() -{ - local PID=$(cat "$XTRABACKUP_PID") - [ -n "$PID" -a $PID -ne 0 ] && kill $PID && (kill $PID && kill -9 $PID) || : - rm -f "$XTRABACKUP_PID" -} - # waits ~10 seconds for nc to open the port and then reports ready # (regardless of timeout) wait_for_listen() From 68eac8a3ad389fef7973190478c0584d792cc88f Mon Sep 17 00:00:00 2001 From: nia Date: Mon, 24 May 2021 15:28:20 +0200 Subject: [PATCH 27/30] my_thread: Use unsigned long long for storing pthread IDs This is a fix for operating systems that have pthread_t defined as a pointer and use the default pthread_self() mechanism for identifying threads. More specifically, this is a build fix for NetBSD. Any changes I submit are freely available under the new BSD license. Signed-off-by: Nia Alarie --- storage/perfschema/my_thread.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/perfschema/my_thread.h b/storage/perfschema/my_thread.h index b9f3f7775ff..2f3a17db2b7 100644 --- a/storage/perfschema/my_thread.h +++ b/storage/perfschema/my_thread.h @@ -17,7 +17,7 @@ typedef pthread_key_t thread_local_key_t; typedef pthread_t my_thread_handle; typedef pthread_attr_t my_thread_attr_t; -typedef uint32 my_thread_os_id_t; +typedef unsigned long long my_thread_os_id_t; #define LOCK_plugin_delete LOCK_plugin @@ -73,7 +73,7 @@ static inline my_thread_os_id_t my_thread_os_id() #else #ifdef HAVE_INTEGER_PTHREAD_SELF /* Unknown platform, fallback. */ - return pthread_self(); + return (unsigned long long)pthread_self(); #else /* Feature not available. */ return 0; From 90adf2aa597acfcfc42d37ae16a73c2d81d70868 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 26 May 2021 09:52:00 +1000 Subject: [PATCH 28/30] perfschema: use glibc gettid if available --- cmake/os/WindowsCache.cmake | 1 + storage/perfschema/CMakeLists.txt | 3 +++ storage/perfschema/my_thread.h | 11 +++++++---- storage/perfschema/pfs_config.h.cmake | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmake/os/WindowsCache.cmake b/cmake/os/WindowsCache.cmake index e9b9e187722..e2a4aa9e520 100644 --- a/cmake/os/WindowsCache.cmake +++ b/cmake/os/WindowsCache.cmake @@ -322,6 +322,7 @@ SET(HAVE_SCHED_GETCPU CACHE INTERNAL "") SET(HAVE_NANOSLEEP CACHE INTERNAL "") SET(HAVE_PTHREAD_THREADID_NP CACHE INTERNAL "") SET(HAVE_SYS_GETTID CACHE INTERNAL "") +SET(HAVE_GETTID CACHE INTERNAL "") SET(HAVE_INTEGER_PTHREAD_SELF CACHE INTERNAL "") SET(HAVE_PTHREAD_GETTHREADID_NP CACHE INTERNAL "") SET(HAVE_TIMER_DELETE CACHE INTERNAL "") diff --git a/storage/perfschema/CMakeLists.txt b/storage/perfschema/CMakeLists.txt index 98d3e2a401d..d5625afbb1a 100644 --- a/storage/perfschema/CMakeLists.txt +++ b/storage/perfschema/CMakeLists.txt @@ -290,6 +290,9 @@ int main(int ac, char **av) }" HAVE_PTHREAD_THREADID_NP) +# gettid() library function (glibc-2.30+) +CHECK_SYMBOL_EXISTS(gettid unistd.h HAVE_GETTID) + # Check for gettid() system call CHECK_C_SOURCE_COMPILES(" #include diff --git a/storage/perfschema/my_thread.h b/storage/perfschema/my_thread.h index 2f3a17db2b7..7f6b1624016 100644 --- a/storage/perfschema/my_thread.h +++ b/storage/perfschema/my_thread.h @@ -50,12 +50,14 @@ static inline my_thread_os_id_t my_thread_os_id() pthread_threadid_np(nullptr, &tid64); return (pid_t)tid64; #else +#ifdef HAVE_GETTID + /* Linux glibc-2.30+ */ + return gettid(); +#else #ifdef HAVE_SYS_GETTID /* - Linux. + Linux before glibc-2.30 See man gettid - See GLIBC Bug 6399 - gettid() should have a wrapper - https://sourceware.org/bugzilla/show_bug.cgi?id=6399 */ return syscall(SYS_gettid); #else @@ -82,7 +84,8 @@ static inline my_thread_os_id_t my_thread_os_id() #endif /* HAVE_PTHREAD_GETTHREADID_NP */ #endif /* _WIN32 */ #endif /* HAVE_SYS_GETTID */ -#endif /* HAVE_SYS_THREAD_SELFID */ +#endif /* HAVE_GETTID */ +#endif /* HAVE_PTHREAD_THREADID_NP */ } #define CHANNEL_NAME_LENGTH MAX_CONNECTION_NAME diff --git a/storage/perfschema/pfs_config.h.cmake b/storage/perfschema/pfs_config.h.cmake index 1b518fe3211..2b61b7e170e 100644 --- a/storage/perfschema/pfs_config.h.cmake +++ b/storage/perfschema/pfs_config.h.cmake @@ -1,5 +1,6 @@ #cmakedefine HAVE_PTHREAD_THREADID_NP 1 #cmakedefine HAVE_SYS_GETTID 1 +#cmakedefine HAVE_GETTID #cmakedefine HAVE_GETTHRID 1 #cmakedefine HAVE_PTHREAD_GETTHREADID_NP 1 #cmakedefine HAVE_INTEGER_PTHREAD_SELF 1 From 0d44792a835128a83ff07f14dbbcdd621df9f7da Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 26 May 2021 10:23:43 +1000 Subject: [PATCH 29/30] perfschema: native type for my_thread_os_id_t Though these will all get case to unsigned long long where it is populated into the perfschema's BIGINT type. Use uintptr_t for NetBSD per Nia Alarie's original #1836. --- storage/perfschema/my_thread.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/storage/perfschema/my_thread.h b/storage/perfschema/my_thread.h index 7f6b1624016..5ddea9b1303 100644 --- a/storage/perfschema/my_thread.h +++ b/storage/perfschema/my_thread.h @@ -17,7 +17,17 @@ typedef pthread_key_t thread_local_key_t; typedef pthread_t my_thread_handle; typedef pthread_attr_t my_thread_attr_t; +#if defined(HAVE_PTHREAD_THREADID_NP) || defined(HAVE_GETTID) || defined(HAVE_SYS_GETTID) || defined(HAVE_GETTHRID) +typedef pid_t my_thread_os_id_t; +#elif defined(_WIN32) +typedef uint32 my_thread_os_id_t; +#elif defined(HAVE_PTHREAD_GETTHREADID_NP) +typedef int my_thread_os_id_t; +#elif defined(HAVE_INTEGER_PTHREAD_SELF) +typedef uintptr_t my_thread_os_id_t; +#else typedef unsigned long long my_thread_os_id_t; +#endif #define LOCK_plugin_delete LOCK_plugin @@ -74,8 +84,8 @@ static inline my_thread_os_id_t my_thread_os_id() return getthrid(); #else #ifdef HAVE_INTEGER_PTHREAD_SELF - /* Unknown platform, fallback. */ - return (unsigned long long)pthread_self(); + /* NetBSD, and perhaps something else, fallback. */ + return (my_thread_os_id_t) pthread_self(); #else /* Feature not available. */ return 0; From 2a4f72b7d2ac30a771a14d097608b0ce282e021c Mon Sep 17 00:00:00 2001 From: Krunal Bauskar Date: Tue, 1 Jun 2021 09:09:56 +1000 Subject: [PATCH 30/30] MDEV-25807: ARM build failure due to missing ISB instruction on ARMv6 Debian has support for 3 different arm machine and probably one which is failing is a pretty old version which doesn't support the said instruction. So we can make it specific to _aarch64_ (as per the arm official reference manual ISB should be defined by all ARMv8 processors). [as per the ARMv7 specs even 32 bits processor should support it but not sure which exact version Debian has under armel]. In either case, the said performance issue will have an impact mainly with a high-end processors with extreme parallelism so it safe to limit it to _aarch64_. Corrects: MDEV-24630 --- include/my_cpu.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/my_cpu.h b/include/my_cpu.h index b42d62e7e82..05fa937c98d 100644 --- a/include/my_cpu.h +++ b/include/my_cpu.h @@ -84,7 +84,15 @@ static inline void MY_RELAX_CPU(void) __ppc_get_timebase(); #elif defined __GNUC__ && (defined __arm__ || defined __aarch64__) /* Mainly, prevent the compiler from optimizing away delay loops */ +#ifdef _aarch64_ __asm__ __volatile__ ("isb":::"memory"); +#else + /* + some older 32 bits processor doesn't support isb but as per + arm-v8 reference manual all armv8 processor should support isb. + */ + __asm__ __volatile__ ("":::"memory"); +#endif #else int32 var, oldval = 0; my_atomic_cas32_strong_explicit(&var, &oldval, 1, MY_MEMORY_ORDER_RELAXED,