From 606c867e7fec4507bd1225f9abf2d0aa664b6ab6 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Wed, 11 Sep 2024 18:53:24 +0200 Subject: [PATCH] galera SST scripts: moving common code to wsrep_sst_common file --- scripts/wsrep_sst_backup.sh | 5 +- scripts/wsrep_sst_common.sh | 102 +++++++++++++++- scripts/wsrep_sst_mariabackup.sh | 89 +------------- scripts/wsrep_sst_rsync.sh | 198 +++++++++++-------------------- 4 files changed, 172 insertions(+), 222 deletions(-) diff --git a/scripts/wsrep_sst_backup.sh b/scripts/wsrep_sst_backup.sh index 6cf17fe7228..af2e396641b 100644 --- a/scripts/wsrep_sst_backup.sh +++ b/scripts/wsrep_sst_backup.sh @@ -21,14 +21,13 @@ set -ue # This is a reference script for rsync-based state snapshot transfer -OS="$(uname)" -[ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH - . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir DATA="$WSREP_SST_OPT_DATA" +create_data + MAGIC_FILE="$DATA/backup_sst_complete" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index b6b8c0501c8..4766d82181d 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -20,8 +20,13 @@ trap 'exit 32' HUP PIPE trap 'exit 3' INT QUIT TERM -# Setting the path for some utilities on CentOS -export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" +OS="$(uname)" + +# Setting the paths for some utilities on CentOS +export PATH="${PATH:+$PATH:}/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin" +if [ "$OS" != 'Darwin' ]; then + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}/usr/local/lib:/usr/lib:/lib:/opt/lib" +fi commandex() { @@ -1724,4 +1729,97 @@ simple_cleanup() exit $estatus } +create_data() +{ + OLD_PWD="$(pwd)" + + if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" + fi + DATA_DIR="$(pwd)" + + cd "$OLD_PWD" +} + +create_dirs() +{ + local simplify=${1:-0} + + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' -a \ + "$INNODB_DATA_HOME_DIR" != "$DATA_DIR" ] + then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" + [ $simplify -ne 0 -a "$ib_home_dir" = "$DATA_DIR" ] && ib_home_dir="" + fi + + # if no command line argument and INNODB_LOG_GROUP_HOME is not set, + # then try to get it from the my.cnf: + if [ -z "$INNODB_LOG_GROUP_HOME" ]; then + INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") + fi + + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' -a \ + "$INNODB_LOG_GROUP_HOME" != "$DATA_DIR" ] + then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" + [ $simplify -ne 0 -a "$ib_log_dir" = "$DATA_DIR" ] && ib_log_dir="" + fi + + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: + if [ -z "$INNODB_UNDO_DIR" ]; then + INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") + fi + + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' -a \ + "$INNODB_UNDO_DIR" != "$DATA_DIR" ] + then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" + [ $simplify -ne 0 -a "$ib_undo_dir" = "$DATA_DIR" ] && ib_undo_dir="" + fi + + # if no command line argument then try to get it from the my.cnf: + if [ -z "$ARIA_LOG_DIR" ]; then + ARIA_LOG_DIR=$(parse_cnf '--mysqld' 'aria-log-dir-path') + ARIA_LOG_DIR=$(trim_dir "$ARIA_LOG_DIR") + fi + + if [ -n "$ARIA_LOG_DIR" -a "$ARIA_LOG_DIR" != '.' -a \ + "$ARIA_LOG_DIR" != "$DATA_DIR" ] + then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$ARIA_LOG_DIR" ] && mkdir -p "$ARIA_LOG_DIR" + cd "$ARIA_LOG_DIR" + ar_log_dir="$(pwd)" + cd "$OLD_PWD" + [ $simplify -ne 0 -a "$ar_log_dir" = "$DATA_DIR" ] && ar_log_dir="" + fi +} + wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE started on $WSREP_SST_OPT_ROLE" diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index c607cb7d7eb..75c1e846c76 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -23,8 +23,6 @@ set -ue # https://mariadb.com/kb/en/mariabackup-overview/ # Make sure to read that before proceeding! -OS="$(uname)" - . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir @@ -935,15 +933,7 @@ if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -OLD_PWD="$(pwd)" - -if [ -n "$DATA" -a "$DATA" != '.' ]; then - [ ! -d "$DATA" ] && mkdir -p "$DATA" - cd "$DATA" -fi -DATA_DIR="$(pwd)" - -cd "$OLD_PWD" +create_data if [ $ssyslog -eq 1 ]; then if [ -n "$(commandex logger)" ]; then @@ -1249,85 +1239,12 @@ if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then else # joiner + create_dirs 1 + [ -e "$SST_PROGRESS_FILE" ] && \ wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - # if no command line argument and INNODB_DATA_HOME_DIR environment - # variable is not set, try to get it from the my.cnf: - if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') - INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") - fi - - if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' -a \ - "$INNODB_DATA_HOME_DIR" != "$DATA_DIR" ] - then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" - ib_home_dir="$(pwd)" - cd "$OLD_PWD" - [ "$ib_home_dir" = "$DATA_DIR" ] && ib_home_dir="" - fi - - # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # then try to get it from the my.cnf: - if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') - INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") - fi - - if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' -a \ - "$INNODB_LOG_GROUP_HOME" != "$DATA_DIR" ] - then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" - cd "$INNODB_LOG_GROUP_HOME" - ib_log_dir="$(pwd)" - cd "$OLD_PWD" - [ "$ib_log_dir" = "$DATA_DIR" ] && ib_log_dir="" - fi - - # if no command line argument and INNODB_UNDO_DIR is not set, - # then try to get it from the my.cnf: - if [ -z "$INNODB_UNDO_DIR" ]; then - INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') - INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") - fi - - if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' -a \ - "$INNODB_UNDO_DIR" != "$DATA_DIR" ] - then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" - cd "$INNODB_UNDO_DIR" - ib_undo_dir="$(pwd)" - cd "$OLD_PWD" - [ "$ib_undo_dir" = "$DATA_DIR" ] && ib_undo_dir="" - fi - - # if no command line argument then try to get it from the my.cnf: - if [ -z "$ARIA_LOG_DIR" ]; then - ARIA_LOG_DIR=$(parse_cnf '--mysqld' 'aria-log-dir-path') - ARIA_LOG_DIR=$(trim_dir "$ARIA_LOG_DIR") - fi - - if [ -n "$ARIA_LOG_DIR" -a "$ARIA_LOG_DIR" != '.' -a \ - "$ARIA_LOG_DIR" != "$DATA_DIR" ] - then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$ARIA_LOG_DIR" ] && mkdir -p "$ARIA_LOG_DIR" - cd "$ARIA_LOG_DIR" - ar_log_dir="$(pwd)" - cd "$OLD_PWD" - [ "$ar_log_dir" = "$DATA_DIR" ] && ar_log_dir="" - fi - if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" fi diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 55b94ff6009..c22d0b3e494 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -21,17 +21,14 @@ set -ue # This is a reference script for rsync-based state snapshot transfer -RSYNC_REAL_PID=0 # rsync process id -STUNNEL_REAL_PID=0 # stunnel process id - -OS="$(uname)" -[ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH - . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir wsrep_check_programs rsync +RSYNC_REAL_PID=0 # rsync process id +STUNNEL_REAL_PID=0 # stunnel process id + cleanup_joiner() { # Since this is invoked just after exit NNN @@ -162,15 +159,7 @@ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi -OLD_PWD="$(pwd)" - -if [ -n "$DATA" -a "$DATA" != '.' ]; then - [ ! -d "$DATA" ] && mkdir -p "$DATA" - cd "$DATA" -fi -DATA_DIR="$(pwd)" - -cd "$OLD_PWD" +create_data BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" @@ -179,77 +168,6 @@ ib_log_dir="$DATA_DIR" ib_home_dir="$DATA_DIR" ib_undo_dir="$DATA_DIR" -# if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# then try to get it from the my.cnf: -if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') - INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") -fi - -if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' -a \ - "$INNODB_LOG_GROUP_HOME" != "$DATA_DIR" ] -then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" - cd "$INNODB_LOG_GROUP_HOME" - ib_log_dir="$(pwd)" - cd "$OLD_PWD" -fi - -# if no command line argument and INNODB_DATA_HOME_DIR environment -# variable is not set, try to get it from the my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') - INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") -fi - -if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' -a \ - "$INNODB_DATA_HOME_DIR" != "$DATA_DIR" ] -then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" - ib_home_dir="$(pwd)" - cd "$OLD_PWD" -fi - -# if no command line argument and INNODB_UNDO_DIR is not set, -# then try to get it from the my.cnf: -if [ -z "$INNODB_UNDO_DIR" ]; then - INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') - INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") -fi - -if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' -a \ - "$INNODB_UNDO_DIR" != "$DATA_DIR" ] -then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" - cd "$INNODB_UNDO_DIR" - ib_undo_dir="$(pwd)" - cd "$OLD_PWD" -fi - -# if no command line argument then try to get it from the my.cnf: -if [ -z "$ARIA_LOG_DIR" ]; then - ARIA_LOG_DIR=$(parse_cnf '--mysqld' 'aria-log-dir-path') - ARIA_LOG_DIR=$(trim_dir "$ARIA_LOG_DIR") -fi - -if [ -n "$ARIA_LOG_DIR" -a "$ARIA_LOG_DIR" != '.' -a \ - "$ARIA_LOG_DIR" != "$DATA_DIR" ] -then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$ARIA_LOG_DIR" ] && mkdir -p "$ARIA_LOG_DIR" - cd "$ARIA_LOG_DIR" - ar_log_dir="$(pwd)" - cd "$OLD_PWD" -fi - encgroups='--mysqld|sst' check_server_ssl_config @@ -598,66 +516,82 @@ FILTER="-f '- /lost+found' wsrep_log_info "Transfer of normal directories done" - # Transfer InnoDB data files - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --dirs --delete --quiet \ - $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$ib_home_dir/" \ - "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? + if [ -d "$ib_home_dir" ]; then + + # Transfer InnoDB data files + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ + -f '- **' "$ib_home_dir/" \ + "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_data_home_dir returned code $RC:" + exit 255 # unknown error + fi + + wsrep_log_info "Transfer of InnoDB data files done" - if [ $RC -ne 0 ]; then - wsrep_log_error "rsync innodb_data_home_dir returned code $RC:" - exit 255 # unknown error fi - wsrep_log_info "Transfer of InnoDB data files done" + if [ -d "$ib_log_dir" ]; then - # second, we transfer InnoDB log files - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --dirs --delete --quiet \ - $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' \ - -f '- **' "$ib_log_dir/" \ - "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? + # second, we transfer InnoDB log files + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' \ + -f '- **' "$ib_log_dir/" \ + "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_log_group_home_dir returned code $RC:" + exit 255 # unknown error + fi + + wsrep_log_info "Transfer of InnoDB log files done" - if [ $RC -ne 0 ]; then - wsrep_log_error "rsync innodb_log_group_home_dir returned code $RC:" - exit 255 # unknown error fi - wsrep_log_info "Transfer of InnoDB log files done" + if [ "$ib_undo_dir" ]; then - # third, we transfer InnoDB undo logs - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --dirs --delete --quiet \ - $WHOLE_FILE_OPT -f '+ /undo*' \ - -f '- **' "$ib_undo_dir/" \ - "rsync://$WSREP_SST_OPT_ADDR-undo_dir" >&2 || RC=$? + # third, we transfer InnoDB undo logs + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /undo*' \ + -f '- **' "$ib_undo_dir/" \ + "rsync://$WSREP_SST_OPT_ADDR-undo_dir" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_undo_dir returned code $RC:" + exit 255 # unknown error + fi + + wsrep_log_info "Transfer of InnoDB undo logs done" - if [ $RC -ne 0 ]; then - wsrep_log_error "rsync innodb_undo_dir returned code $RC:" - exit 255 # unknown error fi - wsrep_log_info "Transfer of InnoDB undo logs done" + if [ "$ar_log_dir" ]; then - # fourth, we transfer Aria logs - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --dirs --delete --quiet \ - $WHOLE_FILE_OPT -f '+ /aria_log_control' -f '+ /aria_log.*' \ - -f '- **' "$ar_log_dir/" \ - "rsync://$WSREP_SST_OPT_ADDR-aria_log" >&2 || RC=$? + # fourth, we transfer Aria logs + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /aria_log_control' -f '+ /aria_log.*' \ + -f '- **' "$ar_log_dir/" \ + "rsync://$WSREP_SST_OPT_ADDR-aria_log" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync aria_log_dir_path returned code $RC:" + exit 255 # unknown error + fi + + wsrep_log_info "Transfer of Aria logs done" - if [ $RC -ne 0 ]; then - wsrep_log_error "rsync aria_log_dir_path returned code $RC:" - exit 255 # unknown error fi - wsrep_log_info "Transfer of Aria logs done" - # then, we parallelize the transfer of database directories, # use '.' so that path concatenation works: @@ -744,6 +678,8 @@ else # joiner check_sockets_utils + create_dirs + ADDR="$WSREP_SST_OPT_HOST" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST"