diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index c46c32213d6..9c6f3665a6a 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -582,7 +582,6 @@ datafile_read(datafile_cur_t *cursor) Check to see if a file exists. Takes name of the file to check. @return true if file exists. */ -static bool file_exists(const char *filename) { @@ -1547,13 +1546,14 @@ bool backup_start(CorruptedPages &corrupted_pages) if (!write_galera_info(mysql_connection)) { return(false); } - write_current_binlog_file(mysql_connection); } - if (opt_binlog_info == BINLOG_INFO_ON) { + bool with_binlogs = opt_binlog_info == BINLOG_INFO_ON; - lock_binlog_maybe(mysql_connection); - write_binlog_info(mysql_connection); + if (with_binlogs || opt_galera_info) { + if (!write_current_binlog_file(mysql_connection, with_binlogs)) { + return(false); + } } if (have_flush_engine_logs && !opt_no_lock) { @@ -1587,15 +1587,34 @@ void backup_release() } } +static const char *default_buffer_pool_file = "ib_buffer_pool"; + +static +const char * get_buffer_pool_filename(size_t *length) +{ + /* If mariabackup is run for Galera, then the file + name is changed to the default so that the receiving + node can find this file and rename it according to its + settings, otherwise we keep the original file name: */ + size_t dir_length = 0; + const char *dst_name = default_buffer_pool_file; + if (!opt_galera_info) { + dir_length = dirname_length(buffer_pool_filename); + dst_name = buffer_pool_filename + dir_length; + } + if (length) { + *length=dir_length; + } + return dst_name; +} + /** Finish after backup_start() and backup_release() */ bool backup_finish() { /* Copy buffer pool dump or LRU dump */ if (!opt_rsync) { if (buffer_pool_filename && file_exists(buffer_pool_filename)) { - const char *dst_name; - - dst_name = trim_dotslash(buffer_pool_filename); + const char *dst_name = get_buffer_pool_filename(NULL); copy_file(ds_data, buffer_pool_filename, dst_name, 0); } if (file_exists("ib_lru_dump")) { @@ -1684,17 +1703,14 @@ ibx_copy_incremental_over_full() /* copy buffer pool dump */ if (innobase_buffer_pool_filename) { - const char *src_name; - - src_name = trim_dotslash(innobase_buffer_pool_filename); + const char *src_name = get_buffer_pool_filename(NULL); snprintf(path, sizeof(path), "%s/%s", xtrabackup_incremental_dir, src_name); if (file_exists(path)) { - copy_file(ds_data, path, - innobase_buffer_pool_filename, 0); + copy_file(ds_data, path, src_name, 0); } } @@ -1929,6 +1945,14 @@ copy_back() datadir_node_init(&node); + /* If mariabackup is run for Galera, then the file + name is changed to the default so that the receiving + node can find this file and rename it according to its + settings, otherwise we keep the original file name: */ + size_t dir_length; + const char *src_buffer_pool; + src_buffer_pool = get_buffer_pool_filename(&dir_length); + while (datadir_iter_next(it, &node)) { const char *ext_list[] = {"backup-my.cnf", "xtrabackup_binary", "xtrabackup_binlog_info", @@ -1991,6 +2015,11 @@ copy_back() continue; } + /* skip buffer pool dump */ + if (!strcmp(filename, src_buffer_pool)) { + continue; + } + /* skip innodb data files */ is_ibdata_file = false; for (Tablespace::const_iterator iter(srv_sys_space.begin()), @@ -2013,23 +2042,18 @@ copy_back() /* copy buffer pool dump */ - if (innobase_buffer_pool_filename) { - const char *src_name; - char path[FN_REFLEN]; - - src_name = trim_dotslash(innobase_buffer_pool_filename); - - snprintf(path, sizeof(path), "%s/%s", - mysql_data_home, - src_name); - - /* could be already copied with other files - from data directory */ - if (file_exists(src_name) && - !file_exists(innobase_buffer_pool_filename)) { - copy_or_move_file(src_name, - innobase_buffer_pool_filename, - mysql_data_home, 0); + if (file_exists(src_buffer_pool)) { + char dst_dir[FN_REFLEN]; + while (IS_TRAILING_SLASH(buffer_pool_filename, dir_length)) { + dir_length--; + } + memcpy(dst_dir, buffer_pool_filename, dir_length); + dst_dir[dir_length] = 0; + if (!(ret = copy_or_move_file(src_buffer_pool, + src_buffer_pool, + dst_dir, 1))) + { + goto cleanup; } } diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index 62b2b1bc232..858182001ce 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -32,6 +32,13 @@ copy_file(ds_ctxt_t *datasink, const char *dst_file_path, uint thread_n); +/************************************************************************ +Check to see if a file exists. +Takes name of the file to check. +@return true if file exists. */ +bool +file_exists(const char *filename); + /** Start --backup */ bool backup_start(CorruptedPages &corrupted_pages); /** Release resources after backup_start() */ diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 824b8b6d5f3..78af2b7f5ac 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -83,7 +83,6 @@ os_event_t kill_query_thread_stop; bool sql_thread_started = false; char *mysql_slave_position = NULL; char *mysql_binlog_position = NULL; -char *buffer_pool_filename = NULL; /* History on server */ time_t history_start_time; @@ -1333,27 +1332,29 @@ cleanup: } +static +bool +write_binlog_info(MYSQL *connection, char *log_bin_dir, + MYSQL_RES *mysql_result, my_ulonglong n_rows, + my_ulonglong start); + /*********************************************************************//** Flush and copy the current binary log file into the backup, if GTID is enabled */ bool -write_current_binlog_file(MYSQL *connection) +write_current_binlog_file(MYSQL *connection, bool write_binlogs) { + char *log_bin = NULL; + char *filename = NULL; + char *position = NULL; char *executed_gtid_set = NULL; char *gtid_binlog_state = NULL; - char *log_bin_file = NULL; char *log_bin_dir = NULL; bool gtid_exists; bool result = true; - char filepath[FN_REFLEN]; - mysql_variable status[] = { - {"Executed_Gtid_Set", &executed_gtid_set}, - {NULL, NULL} - }; - - mysql_variable status_after_flush[] = { - {"File", &log_bin_file}, + mysql_variable log_bin_var[] = { + {"@@GLOBAL.log_bin", &log_bin}, {NULL, NULL} }; @@ -1363,21 +1364,36 @@ write_current_binlog_file(MYSQL *connection) {NULL, NULL} }; + mysql_variable status[] = { + {"File", &filename}, + {"Position", &position}, + {"Executed_Gtid_Set", &executed_gtid_set}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SELECT @@GLOBAL.log_bin", log_bin_var, false); + + /* Do not create xtrabackup_binlog_info if binary log is disabled: */ + if (strncmp(log_bin, "1", 2) != 0) { + goto binlog_disabled; + } + + lock_binlog_maybe(connection); + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + + /* Do not create xtrabackup_binlog_info if replication + has not started yet: */ + if (filename == NULL || position == NULL) { + goto no_replication; + } + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); gtid_exists = (executed_gtid_set && *executed_gtid_set) || (gtid_binlog_state && *gtid_binlog_state); - if (gtid_exists) { - size_t log_bin_dir_length; - - lock_binlog_maybe(connection); - - xb_mysql_query(connection, "FLUSH BINARY LOGS", false); - - read_mysql_variables(connection, "SHOW MASTER STATUS", - status_after_flush, false); + if (write_binlogs || gtid_exists) { if (opt_log_bin != NULL && strchr(opt_log_bin, FN_LIBCHAR)) { /* If log_bin is set, it has priority */ @@ -1387,34 +1403,89 @@ write_current_binlog_file(MYSQL *connection) log_bin_dir = strdup(opt_log_bin); } else if (log_bin_dir == NULL) { /* Default location is MySQL datadir */ - log_bin_dir = strdup("./"); + log_bin_dir = static_cast(malloc(3)); + ut_a(log_bin_dir); + log_bin_dir[0] = '.'; + log_bin_dir[1] = FN_LIBCHAR; + log_bin_dir[2] = 0; } + size_t log_bin_dir_length; + dirname_part(log_bin_dir, log_bin_dir, &log_bin_dir_length); /* strip final slash if it is not the only path component */ - if (log_bin_dir_length > 1 && - log_bin_dir[log_bin_dir_length - 1] == FN_LIBCHAR) { - log_bin_dir[log_bin_dir_length - 1] = 0; + while (IS_TRAILING_SLASH(log_bin_dir, log_bin_dir_length)) { + log_bin_dir_length--; } + log_bin_dir[log_bin_dir_length] = 0; - if (log_bin_dir == NULL || log_bin_file == NULL) { - msg("Failed to get master binlog coordinates from " - "SHOW MASTER STATUS"); + if (log_bin_dir == NULL) { + msg("Failed to locate binary log files"); result = false; goto cleanup; } - snprintf(filepath, sizeof(filepath), "%s%c%s", - log_bin_dir, FN_LIBCHAR, log_bin_file); - result = copy_file(ds_data, filepath, log_bin_file, 0); + uint max_binlogs; + max_binlogs = opt_max_binlogs; + if (max_binlogs == 0) { + if (gtid_exists) { + max_binlogs = 1; + } else { + goto cleanup; + } + } + + xb_mysql_query(connection, "FLUSH BINARY LOGS", false); + + MYSQL_RES *mysql_result; + + mysql_result = xb_mysql_query(connection, "SHOW BINARY LOGS", true); + + ut_ad(mysql_num_fields(mysql_result) >= 2); + + my_ulonglong n_rows; + my_ulonglong start; + + n_rows = mysql_num_rows(mysql_result); + + start = 0; + if (max_binlogs < n_rows) { + start = n_rows - max_binlogs; + } + if (start) { + mysql_data_seek(mysql_result, start); + } + + MYSQL_ROW row; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + char filepath[FN_REFLEN]; + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, binlog_name); + if (file_exists(filepath)) { + result = copy_file(ds_data, filepath, binlog_name, 0); + if (!result) break; + } + } + + if (result) { + write_binlog_info(connection, log_bin_dir, + mysql_result, n_rows, start); + } + + mysql_free_result(mysql_result); } cleanup: - free_mysql_variables(status_after_flush); - free_mysql_variables(status); free_mysql_variables(vars); +no_replication: + free_mysql_variables(status); + +binlog_disabled: + free_mysql_variables(log_bin_var); + return(result); } @@ -1422,8 +1493,11 @@ cleanup: /*********************************************************************//** Retrieves MySQL binlog position and saves it in a file. It also prints it to stdout. */ +static bool -write_binlog_info(MYSQL *connection) +write_binlog_info(MYSQL *connection, char *log_bin_dir, + MYSQL_RES *mysql_result, my_ulonglong n_rows, + my_ulonglong start) { char *filename = NULL; char *position = NULL; @@ -1431,9 +1505,13 @@ write_binlog_info(MYSQL *connection) char *gtid_current_pos = NULL; char *gtid_executed = NULL; char *gtid = NULL; - bool result; + char *buffer; + char *buf; + size_t total; + bool result = true; bool mysql_gtid; bool mariadb_gtid; + bool with_gtid; mysql_variable status[] = { {"File", &filename}, @@ -1451,39 +1529,106 @@ write_binlog_info(MYSQL *connection) read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); read_mysql_variables(connection, "SHOW VARIABLES", vars, true); - if (filename == NULL || position == NULL) { - /* Do not create xtrabackup_binlog_info if binary - log is disabled */ - result = true; - goto cleanup; - } + mysql_gtid = gtid_mode && (strcmp(gtid_mode, "ON") == 0); + mariadb_gtid = gtid_current_pos && *gtid_current_pos; - mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0)); - mariadb_gtid = (gtid_current_pos != NULL); + gtid = (gtid_executed && *gtid_executed) ? gtid_executed : gtid_current_pos; - gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos); - - if (mariadb_gtid || mysql_gtid) { + with_gtid = mariadb_gtid || mysql_gtid; + if (with_gtid) { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s', " "GTID of the last change '%s'", filename, position, gtid) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, - "%s\t%s\t%s\n", filename, position, - gtid); } else { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s'", filename, position) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, - "%s\t%s\n", filename, position); } + mysql_data_seek(mysql_result, start); + + MYSQL_ROW row; + my_ulonglong current; + + total = 1; + current = start; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + /* The position in the current binlog is taken from + the global variable, but for the previous ones it is + determined by their length: */ + const char *binlog_pos = + ++current == n_rows ? position : row[1]; + total += strlen(binlog_name) + strlen(binlog_pos) + 2; + if (with_gtid && current != n_rows) { + /* Add the "\t[]" length to the buffer size: */ + total += 3; + } + } + /* For the last of the binray log files, also add + the length of the GTID (+ one character for '\t'): */ + if (with_gtid) { + total += strlen(gtid) + 1; + } + + buffer = static_cast(malloc(total)); + if (!buffer) { + msg("Failed to allocate memory for temporary buffer"); + result = false; + goto cleanup; + } + + mysql_data_seek(mysql_result, start); + + buf = buffer; + current = start; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + char filepath[FN_REFLEN]; + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, binlog_name); + current++; + if (file_exists(filepath)) { + /* The position in the current binlog is taken from + the global variable, but for the previous ones it is + determined by their length: */ + char *binlog_pos = + current == n_rows ? position : row[1]; + int bytes; + if (with_gtid) { + bytes = snprintf(buf, total, "%s\t%s\t%s\n", + binlog_name, binlog_pos, + current == n_rows ? gtid : "[]"); + } else { + bytes = snprintf(buf, total, "%s\t%s\n", + binlog_name, binlog_pos); + } + if (bytes <= 0) { + goto buffer_overflow; + } + buf += bytes; + total -= bytes; + } + } + + if (buf != buffer) { + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, "%s", buffer); + } + +cleanup2: + free(buffer); + cleanup: - free_mysql_variables(status); free_mysql_variables(vars); + free_mysql_variables(status); return(result); + +buffer_overflow: + msg("Internal error: buffer overflow in the write_binlog_info()"); + result = false; + goto cleanup2; } struct escape_and_quote @@ -1811,7 +1956,6 @@ backup_cleanup() { free(mysql_slave_position); free(mysql_binlog_position); - free(buffer_pool_filename); if (mysql_connection) { mysql_close(mysql_connection); diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index b61fa2362c6..5e78281e1cb 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -28,7 +28,6 @@ extern time_t history_lock_time; extern bool sql_thread_started; extern char *mysql_slave_position; extern char *mysql_binlog_position; -extern char *buffer_pool_filename; /** connection to mysql server */ extern MYSQL *mysql_connection; @@ -62,10 +61,7 @@ void unlock_all(MYSQL *connection); bool -write_current_binlog_file(MYSQL *connection); - -bool -write_binlog_info(MYSQL *connection); +write_current_binlog_file(MYSQL *connection, bool write_binlogs); bool write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h index 1973512ad82..beb49524608 100644 --- a/extra/mariabackup/common.h +++ b/extra/mariabackup/common.h @@ -187,4 +187,14 @@ xb_read_full(File fd, uchar *buf, size_t len) return tlen; } +#ifdef _WIN32 +#define IS_TRAILING_SLASH(name, length) \ + ((length) > 1 && \ + (name[(length) - 1] == '/' || \ + name[(length) - 1] == '\\')) +#else +#define IS_TRAILING_SLASH(name, length) \ + ((length) > 1 && name[(length) - 1] == FN_LIBCHAR) +#endif + #endif diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index bb1e2eaf42d..c1b2a1a9c85 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -240,7 +240,8 @@ long innobase_log_buffer_size = 1024*1024L; long innobase_open_files = 300L; longlong innobase_page_size = (1LL << 14); /* 16KB */ -char* innobase_buffer_pool_filename = NULL; +char *innobase_buffer_pool_filename = NULL; +char *buffer_pool_filename = NULL; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ @@ -347,6 +348,7 @@ uint opt_lock_wait_timeout = 0; uint opt_lock_wait_threshold = 0; uint opt_debug_sleep_before_unlock = 0; uint opt_safe_slave_backup_timeout = 0; +uint opt_max_binlogs = UINT_MAX; const char *opt_history = NULL; @@ -1051,7 +1053,8 @@ enum options_xtrabackup OPT_BACKUP_ROCKSDB, OPT_XTRA_CHECK_PRIVILEGES, OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION, - OPT_INNODB_FORCE_RECOVERY + OPT_INNODB_FORCE_RECOVERY, + OPT_MAX_BINLOGS }; struct my_option xb_client_options[]= { @@ -1454,6 +1457,17 @@ struct my_option xb_client_options[]= { &opt_log_innodb_page_corruption, &opt_log_innodb_page_corruption, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sst_max_binlogs", OPT_MAX_BINLOGS, + "Number of recent binary logs to be included in the backup. " + "Setting this parameter to zero normally disables transmission " + "of binary logs to the joiner nodes during SST using Galera. " + "But sometimes a single current binlog can still be transmitted " + "to the joiner even with sst_max_binlogs=0, because it is " + "required for Galera to work properly with GTIDs support.", + (G_PTR *) &opt_max_binlogs, + (G_PTR *) &opt_max_binlogs, 0, GET_UINT, OPT_ARG, + UINT_MAX, 0, UINT_MAX, 0, 1, 0}, + #define MYSQL_CLIENT #include "sslopt-longopts.h" #undef MYSQL_CLIENT @@ -6272,6 +6286,44 @@ check_all_privileges() } } +static +void +xb_init_buffer_pool(const char * filename) +{ + if (filename && +#ifdef _WIN32 + (filename[0] == '/' || + filename[0] == '\\' || + strchr(filename, ':'))) +#else + filename[0] == FN_LIBCHAR) +#endif + { + buffer_pool_filename = strdup(filename); + } else { + char filepath[FN_REFLEN]; + char *dst_dir = + (innobase_data_home_dir && *innobase_data_home_dir) ? + innobase_data_home_dir : mysql_data_home; + size_t dir_length; + if (dst_dir && *dst_dir) { + dir_length = strlen(dst_dir); + while (IS_TRAILING_SLASH(dst_dir, dir_length)) { + dir_length--; + } + memcpy(filepath, dst_dir, dir_length); + } + else { + filepath[0] = '.'; + dir_length = 1; + } + snprintf(filepath + dir_length, + sizeof(filepath) - dir_length, "%c%s", FN_LIBCHAR, + filename ? filename : "ib_buffer_pool"); + buffer_pool_filename = strdup(filepath); + } +} + bool xb_init() { @@ -6336,11 +6388,15 @@ xb_init() if (!get_mysql_vars(mysql_connection)) { return(false); } + xb_init_buffer_pool(buffer_pool_filename); + if (opt_check_privileges) { check_all_privileges(); } history_start_time = time(NULL); + } else { + xb_init_buffer_pool(innobase_buffer_pool_filename); } return(true); @@ -6634,6 +6690,8 @@ int main(int argc, char **argv) free_error_messages(); mysql_mutex_destroy(&LOCK_error_log); + free(buffer_pool_filename); + if (status == EXIT_SUCCESS) { msg("completed OK!"); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index e2955f58d6a..dbcd0d83f9b 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -69,6 +69,7 @@ extern char *xtrabackup_incremental_dir; extern char *xtrabackup_incremental_basedir; extern char *innobase_data_home_dir; extern char *innobase_buffer_pool_filename; +extern char *buffer_pool_filename; extern char *xb_plugin_dir; extern char *xb_rocksdb_datadir; extern my_bool xb_backup_rocksdb; @@ -164,6 +165,7 @@ extern uint opt_lock_wait_timeout; extern uint opt_lock_wait_threshold; extern uint opt_debug_sleep_before_unlock; extern uint opt_safe_slave_backup_timeout; +extern uint opt_max_binlogs; extern const char *opt_history; diff --git a/mysql-test/suite/galera/include/galera_wsrep_recover.inc b/mysql-test/suite/galera/include/galera_wsrep_recover.inc index d2956ea99e6..aa2f0e2e777 100644 --- a/mysql-test/suite/galera/include/galera_wsrep_recover.inc +++ b/mysql-test/suite/galera/include/galera_wsrep_recover.inc @@ -1,5 +1,12 @@ --echo Performing --wsrep-recover ... +if ($wsrep_recover_additional) +{ +--exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover $wsrep_recover_additional > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 +} +if (!$wsrep_recover_additional) +{ --exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 +} --perl use strict; diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext.result b/mysql-test/suite/galera/r/galera_log_bin_ext.result index f5276b7d1ac..c5ff66d17b3 100644 --- a/mysql-test/suite/galera/r/galera_log_bin_ext.result +++ b/mysql-test/suite/galera/r/galera_log_bin_ext.result @@ -1,4 +1,6 @@ connection node_1; +connection node_2; +connection node_1; reset master; connection node_2; reset master; @@ -40,6 +42,12 @@ hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; COUNT(*) = 2 1 @@ -66,6 +74,7 @@ hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 DROP TABLE t1; DROP TABLE t2; #cleanup diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result new file mode 100644 index 00000000000..c5ff66d17b3 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result @@ -0,0 +1,82 @@ +connection node_1; +connection node_2; +connection node_1; +reset master; +connection node_2; +reset master; +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); +connection node_2; +SELECT COUNT(*) = 1 FROM t1; +COUNT(*) = 1 +1 +SELECT COUNT(*) = 2 FROM t2; +COUNT(*) = 2 +1 +connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 +DROP TABLE t1; +DROP TABLE t2; +#cleanup +connection node_1; +RESET MASTER; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff index 94dd8c2e502..7b10cd23b60 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff @@ -1,6 +1,6 @@ --- galera_sst_rsync.result -+++ galera_sst_rsync,debug.reject -@@ -284,3 +284,111 @@ ++++ galera_sst_rsync.reject +@@ -286,3 +286,111 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff index 525156d88da..23ed9862ea4 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff @@ -1,5 +1,5 @@ ---- suite/galera/r/galera_sst_rsync2.result 2018-09-12 13:09:35.352229478 +0200 -+++ suite/galera/r/galera_sst_rsync2,debug.reject 2018-09-12 17:00:51.601974979 +0200 +--- suite/galera/r/galera_sst_rsync2.result ++++ suite/galera/r/galera_sst_rsync2.reject @@ -286,3 +286,111 @@ DROP TABLE t1; COMMIT; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff index 8b091eb370a..412f8996f15 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff @@ -1,5 +1,5 @@ ---- galera_sst_mariabackup.result -+++ galera_sst_mariabackup,debug.reject +--- galera_sst_xtrabackup-v2.result ++++ galera_sst_xtrabackup-v2.reject @@ -286,5 +286,113 @@ DROP TABLE t1; COMMIT; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff index ac232020037..1cb0725177b 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff @@ -1,11 +1,12 @@ ---- r/galera_sst_xtrabackup-v2_data_dir.result 2018-11-19 12:27:24.795221479 +0200 -+++ r/galera_sst_xtrabackup-v2_data_dir.reject 2018-11-19 19:15:38.774008404 +0200 -@@ -260,3 +260,100 @@ +--- galera_sst_xtrabackup-v2_data_dir.result ++++ galera_sst_xtrabackup-v2_data_dir.reject +@@ -286,5 +286,113 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +while a DDL was in progress on it ++connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; @@ -14,6 +15,7 @@ +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); ++connection node_2; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); @@ -22,9 +24,12 @@ +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; ++connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; ++connection node_2; +SET wsrep_sync_wait = 0; +Killing server ... ++connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); @@ -39,6 +44,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); ++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -46,7 +52,9 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); ++connection node_2; +Performing --wsrep-recover ... ++connection node_2; +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; @@ -57,6 +65,7 @@ +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +COMMIT; ++connection node_1; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); @@ -71,6 +80,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +COMMIT; ++connection node_1a_galera_st_kill_slave_ddl; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -88,6 +98,7 @@ +1 +COMMIT; +SET AUTOCOMMIT=ON; ++connection node_1; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 @@ -101,3 +112,5 @@ +COMMIT; +SET AUTOCOMMIT=ON; +SET GLOBAL debug_dbug = $debug_orig; + disconnect node_2; + disconnect node_1; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result index ff85a7d6c0f..fdb5883b590 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result @@ -286,3 +286,5 @@ COUNT(*) = 0 DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +disconnect node_2; +disconnect node_1; diff --git a/mysql-test/suite/galera/r/galera_wan_restart_sst.result b/mysql-test/suite/galera/r/galera_wan_restart_sst.result index 1adcbfd1d50..2744c91a06b 100644 --- a/mysql-test/suite/galera/r/galera_wan_restart_sst.result +++ b/mysql-test/suite/galera/r/galera_wan_restart_sst.result @@ -1,3 +1,9 @@ +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; +connection node_1; +connection node_2; +connection node_3; +connection node_4; SELECT VARIABLE_VALUE AS EXPECT_4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; EXPECT_4 4 @@ -6,10 +12,8 @@ CREATE TABLE t1 (f1 INTEGER); INSERT INTO t1 VALUES (1); connection node_2; INSERT INTO t1 VALUES (2); -connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; connection node_3; INSERT INTO t1 VALUES (3); -connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; connection node_4; INSERT INTO t1 VALUES (4); connection node_3; @@ -81,3 +85,6 @@ CALL mtr.add_suppression("There are no nodes in the same segment that will ever CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); connection node_4; CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); +connection node_1; +disconnect node_3; +disconnect node_4; diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf index 012209610ea..ae47de90bb8 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf @@ -9,3 +9,6 @@ log-slave-updates log-bin = hostname2-bin log-bin-index = hostname2.bdx log-slave-updates + +[sst] +sst_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.test b/mysql-test/suite/galera/t/galera_log_bin_ext.test index 923bd623a8a..752073aecdb 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.test +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.test @@ -1 +1 @@ ---source galera_log_bin.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf new file mode 100644 index 00000000000..c988136b8fb --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf @@ -0,0 +1,19 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=mariabackup +wsrep_sst_auth="root:" + +[mysqld.1] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.1/data/hostname1-bin +log-bin-index = hostname1.bdx +log-slave-updates + +[mysqld.2] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.2/data/hostname2-bin +log-bin-index = hostname2.bdx +log-slave-updates + +[sst] +transferfmt=@ENV.MTR_GALERA_TFMT +sst_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test new file mode 100644 index 00000000000..47df45b4c71 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test @@ -0,0 +1,2 @@ +--source include/have_mariabackup.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_sst.inc b/mysql-test/suite/galera/t/galera_log_bin_sst.inc new file mode 100644 index 00000000000..5d543f6f8b6 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_sst.inc @@ -0,0 +1,84 @@ +--source include/galera_cluster.inc +--source include/force_restart.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +reset master; +--connection node_2 +reset master; + +# +# Test Galera with --log-bin --log-slave-updates . +# This way the actual MySQL binary log is used, +# rather than Galera's own implementation +# + +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); + +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); + +--connection node_2 +SELECT COUNT(*) = 1 FROM t1; +SELECT COUNT(*) = 2 FROM t2; + +--connection node_1 +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +--let $MASTER_MYPORT=$NODE_MYPORT_1 +--source include/show_binlog_events.inc + +--connection node_2 + +#--connection node_2 +#--source suite/galera/include/galera_stop_replication.inc + +--echo Shutting down server ... +--source include/shutdown_mysqld.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +# +# Force SST +# +--echo Cleaning var directory ... +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mysql +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data + +--connection node_2 + +--echo Starting server ... +let $restart_noprint=2; +--source include/start_mysqld.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +--connection node_2 +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +--let $MASTER_MYPORT=$NODE_MYPORT_2 +--source include/show_binlog_events.inc + +DROP TABLE t1; +DROP TABLE t2; + +--echo #cleanup +--connection node_1 +RESET MASTER; + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync.test b/mysql-test/suite/galera/t/galera_sst_rsync.test index 5c08707e870..c944c8d84b7 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync.test @@ -10,4 +10,5 @@ --source suite/galera/include/galera_st_kill_slave.inc --source suite/galera/include/galera_st_kill_slave_ddl.inc + --source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf index 4f25af7cd8b..3913ab6660f 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf +++ b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf @@ -12,4 +12,3 @@ log_bin wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' log_basename=server2 log_bin - diff --git a/mysql-test/suite/galera/t/galera_wan_restart_sst.test b/mysql-test/suite/galera/t/galera_wan_restart_sst.test index 16e073e7164..2048977b4ef 100644 --- a/mysql-test/suite/galera/t/galera_wan_restart_sst.test +++ b/mysql-test/suite/galera/t/galera_wan_restart_sst.test @@ -12,6 +12,16 @@ --source include/galera_cluster.inc --source include/force_restart.inc +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--let $node_4=node_4 +--source include/auto_increment_offset_save.inc + --let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; --source include/wait_condition.inc SELECT VARIABLE_VALUE AS EXPECT_4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; @@ -23,11 +33,9 @@ INSERT INTO t1 VALUES (1); --connection node_2 INSERT INTO t1 VALUES (2); ---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 --connection node_3 INSERT INTO t1 VALUES (3); ---connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 --connection node_4 INSERT INTO t1 VALUES (4); @@ -156,3 +164,10 @@ CALL mtr.add_suppression("Action message in non-primary configuration from membe --connection node_4 CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc + +--connection node_1 +--disconnect node_3 +--disconnect node_4 diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index deebe7cf820..f1d332a9822 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2012-2015 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -22,6 +22,62 @@ set -ue # Setting the path for some utilities on CentOS export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" +trim_string() +{ + if [ -n "$BASH_VERSION" ]; then + local pattern="[![:space:]${2:-}]" + local x="${1#*$pattern}" + local z=${#1} + x=${#x} + if [ $x -ne $z ]; then + local y="${1%$pattern*}" + y=${#y} + x=$(( z-x-1 )) + y=$(( y-x+1 )) + printf '%s' "${1:$x:$y}" + else + printf '' + fi + else + local pattern="[[:space:]${2:-}]" + echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" + fi +} + +trim_dir() +{ + local t=$(trim_string "$1") + if [ "$t" != '/' ]; then + if [ "${t%/}" != "$t" ]; then + t=$(trim_string "${t%/}") + fi + else + t='.' + fi + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$t" + else + echo "$t" + fi +} + +to_minuses() +{ + local x="$1" + local t="${1#*_}" + local r="" + while [ "$t" != "$x" ]; do + r="$r${x%%_*}-" + x="$t" + t="${t#*_}" + done + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$r$x" + else + echo "$r$x" + fi +} + WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" WSREP_SST_OPT_BINLOG_INDEX="" @@ -44,9 +100,9 @@ WSREP_SST_OPT_ADDR_PORT="" WSREP_SST_OPT_HOST="" WSREP_SST_OPT_HOST_UNESCAPED="" WSREP_SST_OPT_HOST_ESCAPED="" -INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}" -INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}" -INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}" +INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") +INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") +INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") INNODB_FORCE_RECOVERY="" INNOEXTRA="" @@ -138,22 +194,22 @@ case "$1" in ;; '--datadir') # Let's remove the trailing slash: - readonly WSREP_SST_OPT_DATA="${2%/}" + readonly WSREP_SST_OPT_DATA=$(trim_dir "$2") shift ;; '--innodb-data-home-dir') # Let's remove the trailing slash: - readonly INNODB_DATA_HOME_DIR="${2%/}" + readonly INNODB_DATA_HOME_DIR=$(trim_dir "$2") shift ;; '--innodb-log-group-home-dir') # Let's remove the trailing slash: - readonly INNODB_LOG_GROUP_HOME="${2%/}" + readonly INNODB_LOG_GROUP_HOME=$(trim_dir "$2") shift ;; '--innodb-undo-directory') # Let's remove the trailing slash: - readonly INNODB_UNDO_DIR="${2%/}" + readonly INNODB_UNDO_DIR=$(trim_dir "$2") shift ;; '--defaults-file') @@ -247,6 +303,7 @@ case "$1" in '--mysqld-args') original_cmd="" shift + cmd_tail=0 while [ $# -gt 0 ]; do lname="${1#--}" # "--" is interpreted as the end of the list of options: @@ -261,7 +318,7 @@ case "$1" in shift done fi - break; + break fi # Make sure the argument does not start with "--", otherwise it # is a long option, which is processed after this "if": @@ -301,15 +358,25 @@ case "$1" in if [ "${2#-}" = "$2" ]; then shift value="$1" + elif [ "$2" = '--' ]; then + shift + if [ $# -gt 1 ]; then + cmd_tail=1 + shift + value="$1" + fi fi fi - if [ $option = 'h' ]; then + if [ "$option" = 'h' ]; then if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi - elif [ $option != 'u' -a \ - $option != 'P' ] + elif [ "$option" != 'u' -a \ + "$option" != 'P' ] then + if [ $cmd_tail -ne 0 ]; then + option="$option --" + fi if [ -z "$value" ]; then slist="$slist$option" elif [ -z "$slist" ]; then @@ -317,9 +384,16 @@ case "$1" in else slist="$slist -$option '$value'" fi + break + fi + if [ $cmd_tail -ne 0 ]; then + if [ -n "$slist" ]; then + slist="$slist --" + else + slist='-' + fi fi break - else slist="$slist$option" fi @@ -329,7 +403,7 @@ case "$1" in original_cmd="$original_cmd -$slist" fi elif [ -z "$options" ]; then - # We found an equal sign without any characters after it: + # We found an minus sign without any characters after it: original_cmd="$original_cmd -" else # We found a value that does not start with a minus - @@ -338,12 +412,25 @@ case "$1" in original_cmd="$original_cmd '$1'" fi shift - continue; + if [ $cmd_tail -ne 0 ]; then + # All other arguments must be copied unchanged: + while [ $# -gt 0 ]; do + original_cmd="$original_cmd '$1'" + shift + done + break + fi + continue fi # Now we are sure that we are working with an option # that has a "long" name, so remove all characters after # the first equal sign: option="${1%%=*}" + # If the option name contains underscores, then replace + # them to minuses: + if [ "${option#*_}" != "$option" ]; then + option=$(to_minuses "$option") + fi # The "--loose-" prefix should not affect the recognition # of the option name: if [ "${option#--loose-}" != "$option" ]; then @@ -370,19 +457,19 @@ case "$1" in case "$option" in '--innodb-data-home-dir') if [ -z "$INNODB_DATA_HOME_DIR" ]; then - MYSQLD_OPT_INNODB_DATA_HOME_DIR="${value%/}" + MYSQLD_OPT_INNODB_DATA_HOME_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-log-group-home-dir') if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - MYSQLD_OPT_INNODB_LOG_GROUP_HOME="${value%/}" + MYSQLD_OPT_INNODB_LOG_GROUP_HOME=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-undo-directory') if [ -z "$INNODB_UNDO_DIR" ]; then - MYSQLD_OPT_INNODB_UNDO_DIR="${value%/}" + MYSQLD_OPT_INNODB_UNDO_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -412,7 +499,7 @@ case "$1" in ;; '--datadir') if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -471,8 +558,8 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments to mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else # Take the default name: @@ -525,26 +612,23 @@ get_binlog() WSREP_SST_OPT_BINLOG_INDEX=$(parse_cnf '--mysqld' 'log-bin-index') fi # if no command line argument and WSREP_SST_OPT_LOG_BASENAME is not set, - # try to get it from my.cnf: + # then try to get it from my.cnf: if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then WSREP_SST_OPT_LOG_BASENAME=$(parse_cnf '--mysqld' 'log-basename') fi if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - # If the --log-bin option is specified without a parameter, + # If the log-bin option is specified without a parameter, # then we need to build the name of the index file according # to the rules described in the server documentation: - if [ -n "${MYSQLD_OPT_LOG_BIN+x}" -o \ - $(in_config '--mysqld' 'log-bin') -eq 1 ] - then + if [ $(in_config '--mysqld' 'log-bin') -ne 0 ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # If the --log-bin option is present without a value, then - # we take the default name: + # Take the default name: readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -554,13 +638,13 @@ get_binlog() # it according to the specifications for the server: if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then - # If the WSREP_SST_OPT_BINLOG variable is not set, but + # If the WSREP_SST_OPT_BINLOG_INDEX variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG_INDEX equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # the default name (note that base of this name + # Use the default name (note that base of this name # is already defined above): readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi @@ -632,7 +716,7 @@ commandex() # try to use my_print_defaults, mysql and mysqldump that come # with the sources (for MTR suite): script_binary=$(dirname "$0") -SCRIPTS_DIR=$(cd "$script_binary"; pwd -P) +SCRIPTS_DIR=$(cd "$script_binary"; pwd) EXTRA_DIR="$SCRIPTS_DIR/../extra" CLIENT_DIR="$SCRIPTS_DIR/../client" @@ -754,7 +838,11 @@ parse_cnf() if [ -z "$reval" ]; then [ -n "${3:-}" ] && reval="$3" fi - echo "$reval" + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$reval" + else + echo "$reval" + fi } # @@ -804,7 +892,11 @@ in_config() break fi done - echo $found + if [ -n "$BASH_VERSION" ]; then + printf '%s' $found + else + echo $found + fi } wsrep_auth_not_set() @@ -937,11 +1029,22 @@ wsrep_gen_secret() { get_openssl if [ -n "$OPENSSL_BINARY" ]; then - echo $("$OPENSSL_BINARY" rand -hex 16) - else - printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ + "$OPENSSL_BINARY" rand -hex 16 + elif [ -n "$BASH_VERSION" ]; then + printf '%04x%04x%04x%04x%04x%04x%04x%04x' \ $RANDOM $RANDOM $RANDOM $RANDOM \ $RANDOM $RANDOM $RANDOM $RANDOM + elif [ -n "$(commandex cksum)" -a \ + -n "$(commandex printf)" ] + then + printf '%08x%08x%08x%08x' \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) + else + wsrep_log_error "Unable to generate 16-byte secret" + exit 22 fi } @@ -979,14 +1082,14 @@ is_local_ip() if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6]
/": "$ip_util" address show \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \ | grep -F " $1/" >/dev/null && return 0 else local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6]
...": "$ifconfig_util" \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \ | grep -F " $1 " >/dev/null && return 0 fi fi @@ -1049,7 +1152,7 @@ check_port() ss -nlpH "( sport = :$port )" 2>/dev/null | \ grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 else - wsrep_log_error "unknown sockets utility" + wsrep_log_error "Unknown sockets utility" exit 2 # ENOENT fi @@ -1158,13 +1261,6 @@ verify_cert_matches_key() exit 22 fi - # If the diff utility is not installed, then - # we will not do this certificate check: - if [ -z "$(commandex diff)" ]; then - wsrep_log_info "diff utility not found" - return - fi - # If the openssl utility is not installed, then # we will not do this certificate check: get_openssl @@ -1175,9 +1271,9 @@ verify_cert_matches_key() # Generate the public key from the cert and the key. # They should match (otherwise we can't create an SSL connection). - if ! diff <("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null) \ - <("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null) >/dev/null 2>&1 - then + local pk1=$("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null || :) + local pk2=$("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null || :) + if [ "$pk1" != "$pk2" ]; then wsrep_log_error "******************* FATAL ERROR *****************" wsrep_log_error "* The certificate and private key do not match. *" wsrep_log_error "* Please check your certificate and key files. *" @@ -1220,28 +1316,6 @@ check_for_version() return 0 } -trim_string() -{ - if [ -n "$BASH_VERSION" ]; then - local pattern="[![:space:]${2:-}]" - local x="${1#*$pattern}" - local z=${#1} - x=${#x} - if [ $x -ne $z ]; then - local y="${1%$pattern*}" - y=${#y} - x=$(( z-x-1 )) - y=$(( y-x+1 )) - printf '%s' "${1:$x:$y}" - else - printf '' - fi - else - local pattern="[[:space:]${2:-}]" - echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" - fi -} - # # Check whether process is still running. # The first parameter contains the name of the PID file. @@ -1272,6 +1346,10 @@ check_pid() rm -f "$pid_file" || : fi fi + local config="${3:-}" + if [ -n "$config" -a -f "$config" ]; then + rm -f "$config" || : + fi CHECK_PID=0 return 1 } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 471b0e5f5b0..2eedff554e6 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -40,7 +40,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode='DISABLED' sockopt="" progress="" ttime=0 @@ -85,13 +85,13 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' # Required for backup locks # For backup locks it is 1 sent by joiner sst_ver=1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -104,17 +104,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/mariabackup.prepare.log" INNOMOVELOG="$DATA/mariabackup.move.log" INNOBACKUPLOG="$DATA/mariabackup.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -154,7 +151,7 @@ get_keys() return fi - if [ $sfmt = 'tar' ]; then + if [ "$sfmt" = 'tar' ]; then wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 @@ -184,11 +181,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -229,15 +226,15 @@ get_keys() get_transfer() { - if [ $tfmt = 'nc' ]; then + if [ "$tfmt" = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q 'ncat'; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -259,14 +256,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -- '-N'; then + if nc -h 2>&1 | grep -qw -F -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -456,7 +453,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -583,8 +580,14 @@ get_stream() sig_joiner_cleanup() { + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + exit $estatus } cleanup_at_exit() @@ -595,6 +598,8 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -624,7 +629,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -715,7 +720,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -- '-k'; then + if timeout --help | grep -qw -F -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -815,7 +820,9 @@ monitor_process() [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ + "$WSREP_SST_OPT_ROLE" != 'donor' ] +then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -823,25 +830,17 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd)" cd "$OLD_PWD" @@ -869,7 +868,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -929,7 +928,7 @@ setup_commands() recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" } @@ -1052,6 +1051,11 @@ then iopts="--parallel=$backup_threads${iopts:+ }$iopts" fi + max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') + if [ -n "$max_binlogs" ]; then + iopts="--sst-max-binlogs=$max_binlogs${iopts:+ }$iopts" + fi + setup_commands set +e timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -1096,6 +1100,7 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" + wsrep_log_info "mariabackup SST/IST completed on donor" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then @@ -1103,22 +1108,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1142,14 +1178,13 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE='xtrabackup_sst' [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" # May need xtrabackup_checkpoints later on [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" - [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0" ADDR="$WSREP_SST_OPT_ADDR" @@ -1165,7 +1200,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1219,6 +1254,36 @@ then jpid=$! wsrep_log_info "Proceeding with SST" + get_binlog + + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + cd "$DATA" + wsrep_log_info "Cleaning the old binary logs" + # If there is a file with binlogs state, delete it: + [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 + # Clean up the old binlog files and index: + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 || : + done < "$binlog_index" + rm -fv "$binlog_index" 1>&2 + fi + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + cd "$binlog_dir" + if [ "$(pwd)" != "$DATA_DIR" ]; then + wsrep_log_info \ + "Cleaning the binlog directory '$binlog_dir' as well" + fi + fi + rm -fv "$binlog_base".[0-9]* 1>&2 || : + cd "$OLD_PWD" + fi + wsrep_log_info \ "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then @@ -1235,20 +1300,6 @@ then -o -exec rm -rfv {} 1>&2 \+ fi - get_binlog - - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" - fi - fi - TDATA="$DATA" DATA="$DATA/.sst" @@ -1282,11 +1333,13 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + if [ -n "$progress" ] && \ + pv --help | grep -qw -F -- '--line-mode' + then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -- '-F'; then + if pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1296,8 +1349,9 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit 'Joiner-Decompression' \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ + $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1314,25 +1368,9 @@ then fi fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done - cd "$OLD_PWD" - - fi - wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "mariabackup prepare stage" "$INNOAPPLY" + timeit 'mariabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "mariabackup apply finished with errors." \ @@ -1340,10 +1378,43 @@ then exit 22 fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + cd "$DATA_DIR" + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + fi + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + if [ -n "$binlogs" ]; then + wsrep_log_info "Moving binary logs to $binlog_dir" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + mv "$DATA/$bin_file" "$binlog_dir" + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi + cd "$OLD_PWD" + fi + MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "mariabackup move stage" "$INNOMOVE" + timeit 'mariabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" @@ -1370,6 +1441,7 @@ then cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" + wsrep_log_info "mariabackup SST/IST completed on joiner" fi exit 0 diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 1c8fc181328..3c92f489cb5 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -3,7 +3,7 @@ set -ue # Copyright (C) 2009-2015 Codership Oy -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,8 +40,7 @@ then fi # Check client version -if ! $MYSQL_CLIENT --version | grep 'Distrib 10\.[1-9]' >/dev/null -then +if ! $MYSQL_CLIENT --version | grep -q -E 'Distrib 10\.[1-9]'; then $MYSQL_CLIENT --version >&2 wsrep_log_error "this operation requires MySQL client version 10.1 or newer" exit $EINVAL @@ -95,7 +94,7 @@ DROP PREPARE stmt;" SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" SET_WSREP_GTID_DOMAIN_ID="" -if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ]; then +if [ -n "$WSREP_SST_OPT_GTID_DOMAIN_ID" ]; then SET_WSREP_GTID_DOMAIN_ID=" SET @val = (SELECT GLOBAL_VALUE FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES WHERE VARIABLE_NAME = 'WSREP_GTID_STRICT_MODE' AND GLOBAL_VALUE > 0); SET @stmt = IF (@val IS NOT NULL, 'SET GLOBAL WSREP_GTID_DOMAIN_ID=$WSREP_SST_OPT_GTID_DOMAIN_ID', 'SET @dummy = 0'); diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index cc73db5da87..7845f143088 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2010-2014 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -36,6 +36,8 @@ cleanup_joiner() { local failure=0 + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \ "stunnel PID=$STUNNEL_REAL_PID" @@ -58,6 +60,7 @@ cleanup_joiner() if [ $failure -eq 0 ]; then if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" else wsrep_log_warning "rsync cleanup failed." fi @@ -140,66 +143,77 @@ STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" -BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" -BINLOG_N_FILES=1 - get_binlog if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") -fi - -# if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# try to get it from my.cnf: -if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi OLD_PWD="$(pwd)" -WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME" - -cd "$WSREP_SST_OPT_DATA" -if [ -n "$WSREP_LOG_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR" - cd "$WSREP_LOG_DIR" +DATA="$WSREP_SST_OPT_DATA" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -WSREP_LOG_DIR=$(pwd -P) +DATA_DIR="$(pwd)" cd "$OLD_PWD" -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: +BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" + +ib_log_dir="$DATA_DIR" +ib_home_dir="$DATA_DIR" +ib_undo_dir="$DATA_DIR" + +# if no command line argument and INNODB_LOG_GROUP_HOME is not set, +# then try to get it from the my.cnf: +if [ -z "$INNODB_LOG_GROUP_HOME" ]; then + INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") +fi + +if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" +fi + +# if no command line argument and INNODB_DATA_HOME_DIR environment +# variable is not set, try to get it from the my.cnf: if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" fi -INNODB_DATA_HOME_DIR=$(pwd -P) -cd "$OLD_PWD" - -# if no command line argument then try to get it from my.cnf: +# if no command line argument and INNODB_UNDO_DIR is not set, +# then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_UNDO_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" fi -INNODB_UNDO_DIR=$(pwd -P) - -cd "$OLD_PWD" encgroups='--mysqld|sst' @@ -276,7 +290,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" fi if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CHECK_OPT_LOCAL="checkHost = localhost" + CHECK_OPT_LOCAL='checkHost = localhost' fi fi fi @@ -293,14 +307,58 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then fi fi -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' -if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] -then +SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" +# give some time for previous SST to complete: +check_round=0 +while check_pid "$SST_PID" 0; do + wsrep_log_info "Previous SST is not completed, waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 20 ]; then + wsrep_log_error "previous SST script still running." + exit 114 # EALREADY + fi + sleep 1 +done + +echo $$ > "$SST_PID" + +# give some time for stunnel from the previous SST to complete: +check_round=0 +while check_pid "$STUNNEL_PID" 1 "$STUNNEL_CONF"; do + wsrep_log_info "Lingering stunnel daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "stunnel daemon still running." + exit 114 # EALREADY + fi + sleep 1 +done + +MODULE='rsync_sst' +RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" +RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + +# give some time for rsync from the previous SST to complete: +check_round=0 +while check_pid "$RSYNC_PID" 1 "$RSYNC_CONF"; do + wsrep_log_info "Lingering rsync daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "rsync daemon still running." + exit 114 # EALREADY + fi + sleep 1 +done + +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" +[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" + +if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then if [ -n "$STUNNEL" ] then @@ -319,8 +377,6 @@ ${VERIFY_OPT} ${CHECK_OPT} ${CHECK_OPT_LOCAL} EOF - else - [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" fi RC=0 @@ -333,7 +389,7 @@ EOF [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo "flush tables" + echo 'flush tables' # Wait for : # (a) Tables to be flushed, AND @@ -357,32 +413,78 @@ EOF sync - if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ] - then - # Prepare binlog files - cd "$BINLOG_DIRNAME" - - binlog_files_full=$(tail -n $BINLOG_N_FILES \ - "$WSREP_SST_OPT_BINLOG_INDEX") - binlog_files="" - for file in $binlog_files_full; do - binlog_file=$(basename "$file") - binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'" - done - - if [ -n "$binlog_files" ]; then - wsrep_log_info "Preparing binlog files for transfer:" - eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2 + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + # Change the directory to binlog base (if possible): + cd "$DATA" + # Let's check the existence of the file with the index: + if [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ]; then + # Let's read the binlog index: + max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') + if [ -n "$max_binlogs" ]; then + binlog_files="" + if [ $max_binlogs -gt 0 ]; then + binlog_files=$(tail -n $max_binlogs \ + "$WSREP_SST_OPT_BINLOG_INDEX") + fi + else + binlog_files=$(cat "$WSREP_SST_OPT_BINLOG_INDEX") + fi + if [ -n "$binlog_files" ]; then + # Preparing binlog files for transfer: + wsrep_log_info "Preparing binlog files for transfer:" + tar_type=0 + if tar --help | grep -qw -F -- '--transform'; then + tar_type=1 + elif tar --help | grep -qw -E -- '-s[[:space:]]pattern' + then + tar_type=2 + fi + if [ $tar_type -ne 0 ]; then + # Preparing list of the binlog file names: + echo "$binlog_files" | { + binlogs="" + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + binlogs="$binlogs${binlogs:+ }'$bin_file'" + done + if [ -n "$binlogs" ]; then + tar_options='/^.*\///g' + if [ $tar_type -eq 1 ]; then + tar_options="--transform='s$tar_options'" + else + tar_options="-s '$tar_options'" + fi + eval tar -P $tar_options \ + -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2 + fi + } + else + tar_options='-cvf' + echo "$binlog_files" | \ + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + bin_dir=$(dirname "$bin_file") + bin_base=$(basename "$bin_file") + if [ -n "$bin_dir" -a "$bin_dir" != '.' ]; then + tar $tar_options "$BINLOG_TAR_FILE" \ + -C "$bin_dir" "$bin_base" >&2 + else + tar $tar_options "$BINLOG_TAR_FILE" \ + "$bin_base" >&2 + fi + tar_options='-rvf' + done + fi + fi fi - cd "$OLD_PWD" fi - # Use deltaxfer only for WAN + # Use deltaxfer only for WAN: inv=$(basename "$0") WHOLE_FILE_OPT="" if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then - WHOLE_FILE_OPT="--whole-file" + WHOLE_FILE_OPT='--whole-file' fi # Old filter - include everything except selected @@ -399,9 +501,9 @@ FILTER="-f '- /lost+found' -f '- /.pid' -f '- /.conf' -f '+ /wsrep_sst_binlog.tar' - -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '- $INNODB_DATA_HOME_DIR/ibdata*' - -f '+ $INNODB_UNDO_DIR/undo*' + -f '- $ib_home_dir/ib_lru_dump' + -f '- $ib_home_dir/ibdata*' + -f '+ $ib_undo_dir/undo*' -f '+ /*/' -f '- /*'" @@ -435,7 +537,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$INNODB_DATA_HOME_DIR/" \ + -f '- **' "$ib_home_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -448,7 +550,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '+ /aria_log.*' \ - -f '+ /aria_log_control' -f '- **' "$WSREP_LOG_DIR/" \ + -f '+ /aria_log_control' -f '- **' "$ib_log_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -459,7 +561,7 @@ FILTER="-f '- /lost+found' # then, we parallelize the transfer of database directories, # use '.' so that path concatenation works: - cd "$WSREP_SST_OPT_DATA" + cd "$DATA" backup_threads=$(parse_cnf '--mysqld|sst' 'backup-threads') if [ -z "$backup_threads" ]; then @@ -518,68 +620,21 @@ FILTER="-f '- /lost+found' [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" fi + [ -f "$SST_PID" ] && rm -f "$SST_PID" + + wsrep_log_info "rsync SST/IST completed on donor" + elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then check_sockets_utils - SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - - # give some time for previous SST to complete: - check_round=0 - while check_pid "$SST_PID" 0 'wsrep_sst_'; do - wsrep_log_info "previous SST is not completed, waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "previous SST script still running." - exit 114 # EALREADY - fi - sleep 1 - done - - echo $$ > "$SST_PID" - - # give some time for stunnel from the previous SST to complete: - check_round=0 - while check_pid "$STUNNEL_PID" 1; do - wsrep_log_info "Lingering stunnel daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "stunnel daemon already running." - exit 114 # EALREADY - fi - sleep 1 - done - - MODULE="rsync_sst" - RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" - RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" - - # give some time for rsync from the previous SST to complete: - check_round=0 - while check_pid "$RSYNC_PID" 1; do - wsrep_log_info "Lingering rsync daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "rsync daemon already running." - exit 114 # EALREADY - fi - sleep 1 - done - - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - - [ -z "$STUNNEL" -a -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" - ADDR="$WSREP_SST_OPT_ADDR" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" - trap "exit 32" HUP PIPE - trap "exit 3" INT TERM ABRT + trap 'exit 32' HUP PIPE + trap 'exit 3' INT TERM ABRT trap cleanup_joiner EXIT touch "$SST_PROGRESS_FILE" @@ -600,13 +655,11 @@ $SILENT path = $WSREP_SST_OPT_DATA exclude = .zfs [$MODULE-log_dir] - path = $WSREP_LOG_DIR + path = $ib_log_dir [$MODULE-data_dir] - path = $INNODB_DATA_HOME_DIR + path = $ib_home_dir EOF -# rm -rf "$DATA/ib_logfile"* # we don't want old logs around - # If the IP is local, listen only on it: if is_local_ip "$RSYNC_ADDR_UNESCAPED" then @@ -617,7 +670,7 @@ EOF RSYNC_EXTRA_ARGS="" STUNNEL_ACCEPT="$RSYNC_PORT" # Overwrite address with all: - RSYNC_ADDR="*" + RSYNC_ADDR='*' fi if [ -z "$STUNNEL" ]; then @@ -675,11 +728,10 @@ EOF TRANSFER_PID="$STUNNEL_PID" fi - if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ] - then # backward-incompatible behavior + if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then + # backward-incompatible behavior CN="" - if [ -n "$SSTCERT" ] - then + if [ -n "$SSTCERT" ]; then # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then @@ -688,7 +740,7 @@ EOF exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -725,16 +777,53 @@ EOF exit 32 fi + if [ -r "$MAGIC_FILE" ]; then + if [ -n "$MY_SECRET" ]; then + # Check donor supplied secret: + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" + exit 32 + fi + fi + else + # This message should cause joiner to abort: + wsrep_log_info "rsync process ended without creating magic file" + echo "rsync process ended without creating '$MAGIC_FILE'" + exit 32 + fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - if [ -f "$BINLOG_TAR_FILE" ]; then - cd "$BINLOG_DIRNAME" - + binlog_tar_present=0 + [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 + # If it is SST (not an IST) or tar with binlogs is present + # among the transferred files, then we need to remove the + # old binlogs: + if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then + cd "$DATA" + # Clean up the old binlog files and index: binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - - # Clean up old binlog files first - rm -f "$BINLOG_FILENAME".[0-9]* - [ -f "$binlog_index" ] && rm -f "$binlog_index" - + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -f "$bin_file" || : + done < "$binlog_index" + rm -f "$binlog_index" + fi + binlog_cd=0 + # Change the directory to binlog base (if possible): + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + binlog_cd=1 + cd "$binlog_dir" + fi + # Clean up unindexed binlog files: + rm -f "$binlog_base".[0-9]* || : + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + fi + if [ $binlog_tar_present -ne 0 ]; then # Create a temporary file: tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then @@ -744,46 +833,44 @@ EOF else tmpfile=$(TMPDIR="$tmpdir"; mktemp) fi - + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + binlog_cd=0 + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + cd "$binlog_dir" + binlog_cd=1 + fi + # Extracting binlog files: wsrep_log_info "Extracting binlog files:" if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then - wsrep_log_error "Error unpacking tar file with binlog files" rm -f "$tmpfile" + wsrep_log_error "Error unpacking tar file with binlog files" exit 32 fi - # Rebuild binlog index: - while read bin_file; do - echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index" + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" done < "$tmpfile" rm -f "$tmpfile" - cd "$OLD_PWD" fi fi - if [ -r "$MAGIC_FILE" ]; then - if [ -n "$MY_SECRET" ]; then - # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) - if [ "$SECRET" != "$MY_SECRET" ]; then - wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" - exit 32 - fi - # remove secret from the magic file, and output - # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" - else - # Output the UUID:seqno and wsrep_gtid_domain_id: - cat "$MAGIC_FILE" - fi + if [ -n "$MY_SECRET" ]; then + # remove secret from the magic file, and output + # the UUID:seqno & wsrep_gtid_domain_id: + grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" else - # this message should cause joiner to abort - echo "rsync process ended without creating '$MAGIC_FILE'" + # Output the UUID:seqno and wsrep_gtid_domain_id: + cat "$MAGIC_FILE" fi + wsrep_log_info "rsync SST/IST completed on joiner" + # wsrep_cleanup_progress_file # cleanup_joiner else diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 569d5084c98..978c4805205 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -1,5 +1,8 @@ -#!/bin/bash -ue -# Copyright (C) 2017-2021 MariaDB +#!/usr/bin/env bash + +set -ue + +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -38,7 +41,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode='DISABLED' sockopt="" progress="" ttime=0 @@ -85,11 +88,11 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' sst_ver=-1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -102,17 +105,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/innobackupex.prepare.log" INNOMOVELOG="$DATA/innobackupex.move.log" INNOBACKUPLOG="$DATA/innobackupex.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -152,7 +152,7 @@ get_keys() return fi - if [ $sfmt = 'tar' ]; then + if [ "$sfmt" = 'tar' ]; then wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 @@ -182,11 +182,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -231,15 +231,15 @@ get_keys() get_transfer() { - if [ $tfmt = 'nc' ]; then + if [ "$tfmt" = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q 'ncat'; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -261,14 +261,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -- '-N'; then + if nc -h 2>&1 | grep -qw -F -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -458,7 +458,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -592,8 +592,14 @@ get_stream() sig_joiner_cleanup() { + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + exit $estatus } cleanup_at_exit() @@ -604,6 +610,8 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -633,7 +641,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -724,7 +732,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -- '-k'; then + if timeout --help | grep -qw -F -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -824,9 +832,10 @@ monitor_process() # check the version, we require XB-2.3.5 to ensure that we can pass the # datadir via the command-line option -XB_REQUIRED_VERSION="2.3.5" +XB_REQUIRED_VERSION='2.3.5' -XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` +XB_VERSION=$($BACKUP_BIN --version 2>&1 | \ + grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1) if [ -z "$XB_VERSION" ]; then wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version." \ "Needs xtrabackup-$XB_REQUIRED_VERSION or higher to" \ @@ -843,7 +852,9 @@ fi [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ + "$WSREP_SST_OPT_ROLE" != 'donor' ] +then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -851,25 +862,17 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd)" cd "$OLD_PWD" @@ -897,7 +900,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -953,7 +956,7 @@ setup_commands() recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi INNOAPPLY="$BACKUP_BIN$disver$recovery${iapts:+ }$iapts$INNOEXTRA --apply-log${rebuildcmd:+ }$rebuildcmd --datadir='$DATA' '$DATA' $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE" local sfmt_work="$sfmt" if [ "$sfmt" = 'mbstream' ]; then sfmt_work='xbstream' @@ -1124,6 +1127,7 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" + wsrep_log_info "xtrabackup SST/IST completed on donor" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then @@ -1131,22 +1135,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1170,7 +1205,7 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE='xtrabackup_sst' [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" @@ -1193,7 +1228,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1247,6 +1282,36 @@ then jpid=$! wsrep_log_info "Proceeding with SST" + get_binlog + + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + cd "$DATA" + wsrep_log_info "Cleaning the old binary logs" + # If there is a file with binlogs state, delete it: + [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 + # Clean up the old binlog files and index: + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 || : + done < "$binlog_index" + rm -fv "$binlog_index" 1>&2 + fi + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + cd "$binlog_dir" + if [ "$(pwd)" != "$DATA_DIR" ]; then + wsrep_log_info \ + "Cleaning the binlog directory '$binlog_dir' as well" + fi + fi + rm -fv "$binlog_base".[0-9]* 1>&2 || : + cd "$OLD_PWD" + fi + wsrep_log_info \ "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then @@ -1263,20 +1328,6 @@ then -o -exec rm -rfv {} 1>&2 \+ fi - get_binlog - - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" - fi - fi - TDATA="$DATA" DATA="$DATA/.sst" @@ -1312,11 +1363,13 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + if [ -n "$progress" ] && \ + pv --help | grep -qw -F -- '--line-mode' + then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -- '-F'; then + if pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1326,8 +1379,9 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit 'Joiner-Decompression' \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ + $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1344,25 +1398,9 @@ then fi fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done - cd "$OLD_PWD" - - fi - wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "Xtrabackup prepare stage" "$INNOAPPLY" + timeit 'Xtrabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "xtrabackup apply finished with errors." \ @@ -1370,10 +1408,43 @@ then exit 22 fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + cd "$DATA_DIR" + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + fi + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + if [ -n "$binlogs" ]; then + wsrep_log_info "Moving binary logs to $binlog_dir" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + mv "$DATA/$bin_file" "$binlog_dir" + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi + cd "$OLD_PWD" + fi + MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "Xtrabackup move stage" "$INNOMOVE" + timeit 'Xtrabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" @@ -1400,6 +1471,7 @@ then cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" + wsrep_log_info "xtrabackup SST/IST completed on joiner" fi exit 0