From 242b67f1ded480ba97384fdd2a5e36340e402b82 Mon Sep 17 00:00:00 2001 From: Dave Gosselin Date: Fri, 12 Jul 2024 09:15:19 -0400 Subject: [PATCH 01/17] MDEV-27037 mysqlbinlog emits a warning when reaching EOF before stop-condition Emit a warning in the event that we finished processing input files before reaching the boundary indicated by --stop-position. --- client/mysqlbinlog.cc | 20 +++++++++++--- ...nlog_mysqlbinlog_warn_stop_position.result | 13 ++++++++++ ...binlog_mysqlbinlog_warn_stop_position.test | 26 +++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result create mode 100644 mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 3621a3dfef5..ba40e7d597b 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -142,6 +142,7 @@ static char *charset= 0; static uint verbose= 0; static ulonglong start_position, stop_position; +static const longlong stop_position_default= (longlong)(~(my_off_t)0); #define start_position_mot ((my_off_t)start_position) #define stop_position_mot ((my_off_t)stop_position) @@ -1684,8 +1685,8 @@ static struct my_option my_options[] = "Stop reading the binlog at position N. Applies to the last binlog " "passed on the command line.", &stop_position, &stop_position, 0, GET_ULL, - REQUIRED_ARG, (longlong)(~(my_off_t)0), BIN_LOG_HEADER_SIZE, - (ulonglong)(~(my_off_t)0), 0, 0, 0}, + REQUIRED_ARG, stop_position_default, BIN_LOG_HEADER_SIZE, + (ulonglong)stop_position_default, 0, 0, 0}, {"table", 'T', "List entries for just this table (affects only row events).", &table, &table, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -2937,7 +2938,20 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, llstr(old_off,llbuff)); goto err; } - // file->error == 0 means EOF, that's OK, we break in this case + // else file->error == 0 means EOF, that's OK, we break in this case + + /* + Emit a warning in the event that we finished processing input + before reaching the boundary indicated by --stop-position. + */ + if (((longlong)stop_position != stop_position_default) && + stop_position > my_b_tell(file)) + { + retval = OK_STOP; + warning("Did not reach stop position %llu before " + "end of input", stop_position); + } + goto end; } if ((retval= process_event(print_event_info, ev, old_off, logname)) != diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result new file mode 100644 index 00000000000..5b99d30bd84 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result @@ -0,0 +1,13 @@ + +# MDEV-27037 mysqlbinlog emits a warning when reaching EOF before stop-condition + +Case: Default stop position, WARNING must not appear +# MYSQL_BINLOG --short-form --start-position=4 mysql-test/std_data/master-bin.000001 --result-file=warn_pos_test_file.out 2>&1 +Case: Stop position before EOF, WARNING must not appear +# MYSQL_BINLOG --short-form --start-position=4 --stop-position=97 mysql-test/std_data/master-bin.000001 --result-file=warn_pos_test_file.out 2>&1 +Case: Stop position at EOF, WARNING must not appear +# MYSQL_BINLOG --short-form --start-position=4 --stop-position=98 mysql-test/std_data/master-bin.000001 --result-file=warn_pos_test_file.out 2>&1 +Case: Stop position after EOF, WARNING must appear +# MYSQL_BINLOG --short-form --start-position=4 --stop-position=99 mysql-test/std_data/master-bin.000001 --result-file=warn_pos_test_file.out 2>&1 +WARNING: Did not reach stop position 99 before end of input +# End of binlog_mysqlbinlog_warn_stop_position.test diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test new file mode 100644 index 00000000000..97fabfc6513 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test @@ -0,0 +1,26 @@ +--echo +--echo # MDEV-27037 mysqlbinlog emits a warning when reaching EOF before stop-condition +--echo + +--let assert_file= $MYSQLTEST_VARDIR/tmp/warn_pos_test_file.out +--let data_file= $MYSQLTEST_VARDIR/std_data/master-bin.000001 + +--echo Case: Default stop position, WARNING must not appear +--echo # MYSQL_BINLOG --short-form --start-position=4 mysql-test/std_data/master-bin.000001 --result-file=warn_pos_test_file.out 2>&1 +--exec $MYSQL_BINLOG --short-form --start-position=4 $data_file --result-file=$assert_file 2>&1 + +--echo Case: Stop position before EOF, WARNING must not appear +--echo # MYSQL_BINLOG --short-form --start-position=4 --stop-position=97 mysql-test/std_data/master-bin.000001 --result-file=warn_pos_test_file.out 2>&1 +--exec $MYSQL_BINLOG --short-form --start-position=4 --stop-position=97 $data_file --result-file=$assert_file 2>&1 + +--echo Case: Stop position at EOF, WARNING must not appear +--echo # MYSQL_BINLOG --short-form --start-position=4 --stop-position=98 mysql-test/std_data/master-bin.000001 --result-file=warn_pos_test_file.out 2>&1 +--exec $MYSQL_BINLOG --short-form --start-position=4 --stop-position=98 $data_file --result-file=$assert_file 2>&1 + +--echo Case: Stop position after EOF, WARNING must appear +--echo # MYSQL_BINLOG --short-form --start-position=4 --stop-position=99 mysql-test/std_data/master-bin.000001 --result-file=warn_pos_test_file.out 2>&1 +--exec $MYSQL_BINLOG --short-form --start-position=4 --stop-position=99 $data_file --result-file=$assert_file 2>&1 + +--remove_file $assert_file + +--echo # End of binlog_mysqlbinlog_warn_stop_position.test From 95885261f0ff70e563140f5fb94da134b84d79b4 Mon Sep 17 00:00:00 2001 From: Dave Gosselin Date: Mon, 22 Jul 2024 11:13:06 -0400 Subject: [PATCH 02/17] MDEV-27037 mysqlbinlog emits a warning when reaching EOF before stop-datetime Emit a warning in the event that we finished processing input files before reaching the boundary indicated by --stop-datetime. --- client/mysqlbinlog.cc | 14 +++++++ ...nlog_mysqlbinlog_warn_stop_datetime.result | 23 ++++++++++ ...binlog_mysqlbinlog_warn_stop_datetime.test | 42 +++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_datetime.result create mode 100644 mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_datetime.test diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index ba40e7d597b..5762e3965b5 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -2847,6 +2847,7 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, IO_CACHE cache,*file= &cache; uchar tmp_buff[BIN_LOG_HEADER_SIZE]; Exit_status retval= OK_CONTINUE; + my_time_t last_ev_when= MY_TIME_T_MAX; if (logname && strcmp(logname, "-") != 0) { @@ -2952,8 +2953,21 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, "end of input", stop_position); } + /* + Emit a warning in the event that we finished processing input + before reaching the boundary indicated by --stop-datetime. + */ + if (stop_datetime != MY_TIME_T_MAX && + stop_datetime > last_ev_when) + { + retval = OK_STOP; + warning("Did not reach stop datetime '%s' " + "before end of input", stop_datetime_str); + } + goto end; } + last_ev_when= ev->when; if ((retval= process_event(print_event_info, ev, old_off, logname)) != OK_CONTINUE) goto end; diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_datetime.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_datetime.result new file mode 100644 index 00000000000..6ce0f2d63e5 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_datetime.result @@ -0,0 +1,23 @@ + +# MDEV-27037 mysqlbinlog emits a warning when reaching EOF before stop-datetime + +set timestamp=1000000000; +CREATE TABLE t1(word VARCHAR(20)); +set timestamp=1000000010; +INSERT INTO t1 VALUES ("abirvalg"); +set timestamp=1000000020; +INSERT INTO t1 SELECT * FROM t1; +flush logs; +Case: Default, must not see warning. +# MYSQL_BINLOG --short-form MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +Case: Stop datetime before EOF, must not see warning. +# MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:50' MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +Case: Stop datetime between records, must not see warning. +# MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:55' MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +Case: Stop datetime at EOF, must not see warning. +# MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:55' MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +Case: Stop datetime after EOF, must see warning. +# MYSQL_BINLOG --short-form --stop-datetime='2035-01-19 03:14:05' MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +WARNING: Did not reach stop datetime '2035-01-19 03:14:05' before end of input +DROP TABLE t1; +# End of binlog_mysqlbinlog_warn_stop_datetime.test diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_datetime.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_datetime.test new file mode 100644 index 00000000000..9d30544c5f4 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_datetime.test @@ -0,0 +1,42 @@ +--echo +--echo # MDEV-27037 mysqlbinlog emits a warning when reaching EOF before stop-datetime +--echo + +--source include/have_binlog_format_statement.inc + +--let ignored_output_file= $MYSQLTEST_VARDIR/tmp/warn_pos_test_file.out + +set timestamp=1000000000; +CREATE TABLE t1(word VARCHAR(20)); +set timestamp=1000000010; +INSERT INTO t1 VALUES ("abirvalg"); +set timestamp=1000000020; +INSERT INTO t1 SELECT * FROM t1; +--let MYSQLD_DATADIR= `select @@datadir;` +flush logs; + +--echo Case: Default, must not see warning. +--echo # MYSQL_BINLOG --short-form MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +--exec $MYSQL_BINLOG --short-form $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1 + +--echo Case: Stop datetime before EOF, must not see warning. +--echo # MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:50' MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +--exec $MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:50' $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1 + +--echo Case: Stop datetime between records, must not see warning. +--echo # MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:55' MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +--exec $MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:55' $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1 + +--echo Case: Stop datetime at EOF, must not see warning. +--echo # MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:55' MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +--exec $MYSQL_BINLOG --short-form --stop-datetime='2001-09-08 21:46:55' $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1 + +--echo Case: Stop datetime after EOF, must see warning. +--echo # MYSQL_BINLOG --short-form --stop-datetime='2035-01-19 03:14:05' MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file +--exec $MYSQL_BINLOG --short-form --stop-datetime='2035-01-19 03:14:05' $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1 + +DROP TABLE t1; + +--remove_file $ignored_output_file + +--echo # End of binlog_mysqlbinlog_warn_stop_datetime.test From b331cde26be8d6db73f530a8689fe5b79dec9d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 13 Sep 2024 14:34:08 +0300 Subject: [PATCH 03/17] MDEV-34921 MemorySanitizer reports errors for non-debug builds my_b_encr_write(): Initialize also block_length, and at the same time last_block_length, so that all 128 bits can be initialized with fewer writes. This fixes an error that was caught in the test encryption.tempfiles_encrypted. test_my_safe_print_str(): Skip a test that would attempt to display uninitialized data in the test unit.stacktrace. Previously, our CI did not build unit tests with MemorySanitizer. handle_delayed_insert(): Remove a redundant call to pthread_exit(0), which would for some reason cause MemorySanitizer in clang-19 to report a stack overflow in a RelWithDebInfo build. This fixes a failure of several tests. Reviewed by: Vladislav Vaintroub --- sql/mf_iocache_encr.cc | 4 ++-- sql/sql_insert.cc | 1 - unittest/mysys/stacktrace-t.c | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sql/mf_iocache_encr.cc b/sql/mf_iocache_encr.cc index 63830ec620a..3175226a3d1 100644 --- a/sql/mf_iocache_encr.cc +++ b/sql/mf_iocache_encr.cc @@ -175,8 +175,8 @@ static int my_b_encr_write(IO_CACHE *info, const uchar *Buffer, size_t Count) DBUG_RETURN(info->error= -1); } crypt_data->counter= 0; - - IF_DBUG(crypt_data->block_length= 0,); + crypt_data->block_length= 0; + crypt_data->last_block_length= 0; } do diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 46486e09a8b..875ae9f9c63 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -3498,7 +3498,6 @@ pthread_handler_t handle_delayed_insert(void *arg) DBUG_LEAVE; } my_thread_end(); - pthread_exit(0); return 0; } diff --git a/unittest/mysys/stacktrace-t.c b/unittest/mysys/stacktrace-t.c index d8408f80d76..c8b699ca696 100644 --- a/unittest/mysys/stacktrace-t.c +++ b/unittest/mysys/stacktrace-t.c @@ -30,7 +30,7 @@ void test_my_safe_print_str() memcpy(b_bss, "LEGAL", 6); #ifdef HAVE_STACKTRACE -#ifndef __SANITIZE_ADDRESS__ +# if !defined __SANITIZE_ADDRESS__ && !__has_feature(memory_sanitizer) fprintf(stderr, "\n===== stack =====\n"); my_safe_print_str(b_stack, 65535); fprintf(stderr, "\n===== heap =====\n"); @@ -40,15 +40,15 @@ void test_my_safe_print_str() fprintf(stderr, "\n===== data =====\n"); my_safe_print_str("LEGAL", 65535); fprintf(stderr, "\n===== Above is a junk, but it is expected. =====\n"); -#endif /*__SANITIZE_ADDRESS__*/ +# endif fprintf(stderr, "\n===== Nornal length test =====\n"); my_safe_print_str("LEGAL", 5); fprintf(stderr, "\n===== NULL =====\n"); my_safe_print_str(0, 5); -#ifndef __SANITIZE_ADDRESS__ +# ifndef __SANITIZE_ADDRESS__ fprintf(stderr, "\n===== (const char*) 1 =====\n"); my_safe_print_str((const char*)1, 5); -#endif /*__SANITIZE_ADDRESS__*/ +# endif /*__SANITIZE_ADDRESS__*/ #endif /*HAVE_STACKTRACE*/ free(b_heap); From 4010dff058d8fc88d602e3bc231a6558599b89f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 14 Sep 2024 11:05:44 +0300 Subject: [PATCH 04/17] mtr_t::log_file_op(): Fix -Wnonnull GCC 12.2.0 could issue -Wnonnull for an unreachable call to strlen(new_path). Let us prevent that by replacing the condition (type == FILE_RENAME) with the equivalent (new_path). This should also optimize the generated code, because the life time of the parameter "type" will be reduced. --- storage/innobase/fil/fil0fil.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index ba4fbc49204..514d6f51fff 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1554,17 +1554,18 @@ inline void mtr_t::log_file_op(mfile_type_t type, ulint space_id, m_last= nullptr; const size_t len= strlen(path); - const size_t new_len= type == FILE_RENAME ? 1 + strlen(new_path) : 0; + const size_t new_len= new_path ? 1 + strlen(new_path) : 0; ut_ad(len > 0); byte *const log_ptr= m_log.open(1 + 3/*length*/ + 5/*space_id*/ + 1/*page_no=0*/); + *log_ptr= type; byte *end= log_ptr + 1; end= mlog_encode_varint(end, space_id); *end++= 0; - if (UNIV_LIKELY(end + len + new_len >= &log_ptr[16])) + const byte *const final_end= end + len + new_len; + if (UNIV_LIKELY(final_end >= &log_ptr[16])) { - *log_ptr= type; - size_t total_len= len + new_len + end - log_ptr - 15; + size_t total_len= final_end - log_ptr - 15; if (total_len >= MIN_3BYTE) total_len+= 2; else if (total_len >= MIN_2BYTE) @@ -1575,13 +1576,13 @@ inline void mtr_t::log_file_op(mfile_type_t type, ulint space_id, } else { - *log_ptr= static_cast(type | (end + len + new_len - &log_ptr[1])); + *log_ptr= static_cast(*log_ptr | (final_end - &log_ptr[1])); ut_ad(*log_ptr & 15); } m_log.close(end); - if (type == FILE_RENAME) + if (new_path) { ut_ad(strchr(new_path, OS_PATH_SEPARATOR)); m_log.push(reinterpret_cast(path), uint32_t(len + 1)); From 7ee0e60bbb861715b8d943683cc89ddf57b73f2e Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 3 Sep 2024 07:39:30 +0200 Subject: [PATCH 05/17] galera mtr tests: minor fixes to make tests more reliable --- mysql-test/suite/galera/r/GCF-939.result | 6 +++++- mysql-test/suite/galera/t/GCF-939.test | 6 +++++- mysql-test/suite/galera/t/galera_fk_lock_wait.cnf | 11 +++++++++++ mysql-test/suite/galera/t/galera_fk_lock_wait.test | 1 + mysql-test/suite/galera_3nodes/r/GCF-354.result | 9 +++------ .../galera_3nodes/r/galera_safe_to_bootstrap.result | 2 ++ mysql-test/suite/galera_3nodes/t/GCF-354.test | 10 ++++++---- .../galera_3nodes/t/galera_safe_to_bootstrap.test | 2 ++ 8 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 mysql-test/suite/galera/t/galera_fk_lock_wait.cnf diff --git a/mysql-test/suite/galera/r/GCF-939.result b/mysql-test/suite/galera/r/GCF-939.result index f9b65bced42..2e94fc4a1b0 100644 --- a/mysql-test/suite/galera/r/GCF-939.result +++ b/mysql-test/suite/galera/r/GCF-939.result @@ -5,7 +5,11 @@ DROP TABLE t1; ERROR 42S02: Unknown table 'test.t1' CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); -GRA_.log +connection node_2; +SELECT * FROM t1; +f1 +1 +connection node_1; GRA_.log DROP TABLE t1; CALL mtr.add_suppression("Ignoring error 'Unknown table 'test\\.t1'' on query"); diff --git a/mysql-test/suite/galera/t/GCF-939.test b/mysql-test/suite/galera/t/GCF-939.test index 62bcdb818c7..a9b9077d5ef 100644 --- a/mysql-test/suite/galera/t/GCF-939.test +++ b/mysql-test/suite/galera/t/GCF-939.test @@ -13,6 +13,11 @@ DROP TABLE t1; CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); +--connection node_2 +SELECT * FROM t1; + +--connection node_1 + # Expect only one GRA_*.log file # TODO replace_regex is somehow broken, it will filter out # result totally if replacement string is already in result @@ -20,7 +25,6 @@ INSERT INTO t1 VALUES (1); # to get GRA_.log two times, this works for some reason # --replace_regex /GRA_.+\.log/GRA_.log/ ---list_files $MYSQLTEST_VARDIR/mysqld.2/data GRA_*.log --replace_regex /GRA_.+\.log/GRA_.log/ --list_files $MYSQLTEST_VARDIR/mysqld.2/data GRA_*.log diff --git a/mysql-test/suite/galera/t/galera_fk_lock_wait.cnf b/mysql-test/suite/galera/t/galera_fk_lock_wait.cnf new file mode 100644 index 00000000000..7df8ed28a4d --- /dev/null +++ b/mysql-test/suite/galera/t/galera_fk_lock_wait.cnf @@ -0,0 +1,11 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +wsrep-debug=1 +auto_increment_offset=1 +auto_increment_increment=2 + +[mysqld.2] +wsrep-debug=1 +auto_increment_offset=2 +auto_increment_increment=2 diff --git a/mysql-test/suite/galera/t/galera_fk_lock_wait.test b/mysql-test/suite/galera/t/galera_fk_lock_wait.test index 150c7397f7e..0d731e154f2 100644 --- a/mysql-test/suite/galera/t/galera_fk_lock_wait.test +++ b/mysql-test/suite/galera/t/galera_fk_lock_wait.test @@ -1,4 +1,5 @@ --source include/galera_cluster.inc +--source include/have_innodb.inc CREATE TABLE parent(parent_id int not null AUTO_INCREMENT PRIMARY KEY, parent_name varchar(80)) ENGINE=InnoDB; diff --git a/mysql-test/suite/galera_3nodes/r/GCF-354.result b/mysql-test/suite/galera_3nodes/r/GCF-354.result index c69de54747f..3fdd44fe9d3 100644 --- a/mysql-test/suite/galera_3nodes/r/GCF-354.result +++ b/mysql-test/suite/galera_3nodes/r/GCF-354.result @@ -1,20 +1,17 @@ connection node_2; connection node_1; -connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; connection node_1; connection node_2; connection node_3; connection node_2; -SET wsrep_on=OFF; +SET SESSION wsrep_on=OFF; DROP SCHEMA test; connection node_3; -SET wsrep_on=OFF; +SET SESSION wsrep_on=OFF; CREATE TABLE test.t1 (f1 INTEGER NOT NULL PRIMARY KEY) engine=innodb; connection node_1; CREATE TABLE test.t1 (f1 INTEGER NOT NULL PRIMARY KEY) engine=innodb; -SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; -VARIABLE_VALUE -1 +INSERT INTO test.t1 values (1); SHOW STATUS LIKE 'wsrep_cluster_status'; Variable_name Value wsrep_cluster_status Primary diff --git a/mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result b/mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result index 2aefefaad00..e966a36fb79 100644 --- a/mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result +++ b/mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result @@ -45,6 +45,7 @@ CALL mtr.add_suppression("It may not be safe to bootstrap the cluster from this CALL mtr.add_suppression("Aborting"); CALL mtr.add_suppression("Plugin 'wsrep' init function returned error\\."); CALL mtr.add_suppression("Plugin 'wsrep' registration as a STORAGE ENGINE failed\\."); +CALL mtr.add_suppression("Plugin 'wsrep' registration as a FUNCTION failed\\."); CALL mtr.add_suppression("Failed to initialize plugins\\."); CALL mtr.add_suppression("WSREP: gcs/src/gcs_core.cpp:core_handle_uuid_msg\\(\\)"); connection node_3; @@ -58,6 +59,7 @@ CALL mtr.add_suppression("It may not be safe to bootstrap the cluster from this CALL mtr.add_suppression("Aborting"); CALL mtr.add_suppression("Plugin 'wsrep' init function returned error\\."); CALL mtr.add_suppression("Plugin 'wsrep' registration as a STORAGE ENGINE failed\\."); +CALL mtr.add_suppression("Plugin 'wsrep' registration as a FUNCTION failed\\."); CALL mtr.add_suppression("Failed to initialize plugins\\."); CALL mtr.add_suppression("WSREP: gcs/src/gcs_core.cpp:core_handle_uuid_msg\\(\\)"); SHOW CREATE TABLE t1; diff --git a/mysql-test/suite/galera_3nodes/t/GCF-354.test b/mysql-test/suite/galera_3nodes/t/GCF-354.test index 372f6329e8e..44dfa3deeb7 100644 --- a/mysql-test/suite/galera_3nodes/t/GCF-354.test +++ b/mysql-test/suite/galera_3nodes/t/GCF-354.test @@ -2,7 +2,9 @@ --source include/have_innodb.inc --source include/force_restart.inc ---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--let $galera_connection_name = node_3 +--let $galera_server_number = 3 +--source include/galera_connect.inc # Save original auto_increment_offset values. --let $node_1=node_1 @@ -14,11 +16,11 @@ # 1. Create different inconsistencies on nodes 2 and 3 # --connection node_2 -SET wsrep_on=OFF; +SET SESSION wsrep_on=OFF; DROP SCHEMA test; --connection node_3 -SET wsrep_on=OFF; +SET SESSION wsrep_on=OFF; CREATE TABLE test.t1 (f1 INTEGER NOT NULL PRIMARY KEY) engine=innodb; # # 2. The following should generate different errors on nodes 2 and 3 and @@ -27,10 +29,10 @@ CREATE TABLE test.t1 (f1 INTEGER NOT NULL PRIMARY KEY) engine=innodb; # --connection node_1 CREATE TABLE test.t1 (f1 INTEGER NOT NULL PRIMARY KEY) engine=innodb; +INSERT INTO test.t1 values (1); --let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' --source include/wait_condition.inc -SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status' --source include/wait_condition.inc SHOW STATUS LIKE 'wsrep_cluster_status'; diff --git a/mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test b/mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test index 80482151518..eb41137b8b3 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test +++ b/mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test @@ -193,6 +193,7 @@ CALL mtr.add_suppression("It may not be safe to bootstrap the cluster from this CALL mtr.add_suppression("Aborting"); CALL mtr.add_suppression("Plugin 'wsrep' init function returned error\\."); CALL mtr.add_suppression("Plugin 'wsrep' registration as a STORAGE ENGINE failed\\."); +CALL mtr.add_suppression("Plugin 'wsrep' registration as a FUNCTION failed\\."); CALL mtr.add_suppression("Failed to initialize plugins\\."); CALL mtr.add_suppression("WSREP: gcs/src/gcs_core.cpp:core_handle_uuid_msg\\(\\)"); @@ -207,6 +208,7 @@ CALL mtr.add_suppression("It may not be safe to bootstrap the cluster from this CALL mtr.add_suppression("Aborting"); CALL mtr.add_suppression("Plugin 'wsrep' init function returned error\\."); CALL mtr.add_suppression("Plugin 'wsrep' registration as a STORAGE ENGINE failed\\."); +CALL mtr.add_suppression("Plugin 'wsrep' registration as a FUNCTION failed\\."); CALL mtr.add_suppression("Failed to initialize plugins\\."); CALL mtr.add_suppression("WSREP: gcs/src/gcs_core.cpp:core_handle_uuid_msg\\(\\)"); From 5cb436e07b0fa46ae935c58519c044293b54f66b Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 10 Sep 2024 02:44:46 +0200 Subject: [PATCH 06/17] MDEV-30822 preparation: refactoring galera sst scripts This commit makes the SST script for mariabackup more resilient to unexpected terminations or hangs while mariabackup or when SST scripts in a previous session are still running (in reality they were hung while waiting for something). --- scripts/wsrep_sst_backup.sh | 2 +- scripts/wsrep_sst_mariabackup.sh | 55 +++++++++++++++++--------------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/scripts/wsrep_sst_backup.sh b/scripts/wsrep_sst_backup.sh index 6f8c658135a..a28dcf494b2 100644 --- a/scripts/wsrep_sst_backup.sh +++ b/scripts/wsrep_sst_backup.sh @@ -33,7 +33,7 @@ export PATH="/usr/sbin:/sbin:$PATH" . $(dirname "$0")/wsrep_sst_common MAGIC_FILE="$WSREP_SST_OPT_DATA/backup_sst_complete" -rm -rf "$MAGIC_FILE" +rm -r "$MAGIC_FILE" WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} # if WSREP_LOG_DIR env. variable is not set, try to get it from my.cnf diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 6eea14df5d2..de219283d15 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -106,6 +106,7 @@ DATA="$WSREP_SST_OPT_DATA" INFO_FILE='xtrabackup_galera_info' DONOR_INFO_FILE='donor_galera_info' IST_FILE='xtrabackup_ist' + MAGIC_FILE="$DATA/$INFO_FILE" DONOR_MAGIC_FILE="$DATA/$DONOR_INFO_FILE" @@ -919,9 +920,6 @@ monitor_process() done } -[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -[ -f "$DONOR_MAGIC_FILE" ] && rm -rf "$DONOR_MAGIC_FILE" - read_cnf setup_ports @@ -1071,6 +1069,24 @@ get_transfer findopt='-L' [ "$OS" = 'FreeBSD' ] && findopt="$findopt -E" +SST_PID="$DATA/wsrep_sst.pid" + +# give some time for previous SST to complete: +check_round=0 +while check_pid "$SST_PID" 0; do + wsrep_log_info "previous SST is not completed, waiting for it to exit" + check_round=$(( check_round+1 )) + if [ $check_round -eq 30 ]; then + wsrep_log_error "previous SST script still running." + exit 114 # EALREADY + fi + sleep 1 +done + +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" +[ -f "$DONOR_MAGIC_FILE" ] && rm -f "$DONOR_MAGIC_FILE" +[ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" + if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then trap cleanup_at_exit EXIT @@ -1319,33 +1335,12 @@ else # joiner impts="--parallel=$backup_threads${impts:+ }$impts" fi - SST_PID="$DATA/wsrep_sst.pid" - - # give some time for previous SST to complete: - check_round=0 - while check_pid "$SST_PID" 0; do - wsrep_log_info "previous SST is not completed, waiting for it to exit" - check_round=$(( check_round+1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "previous SST script still running." - exit 114 # EALREADY - fi - sleep 1 - done - trap simple_cleanup EXIT echo $$ > "$SST_PID" stagemsg='Joiner-Recv' MODULE="${WSREP_SST_OPT_MODULE:-xtrabackup_sst}" - - [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" - - # May need xtrabackup_checkpoints later on - [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" - [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" - ADDR="$WSREP_SST_OPT_HOST" if [ "${tmode#VERIFY}" != "$tmode" ]; then @@ -1376,6 +1371,7 @@ else # joiner STATDIR="$(mktemp -d)" MAGIC_FILE="$STATDIR/$INFO_FILE" + DONOR_MAGIC_FILE="$STATDIR/$DONOR_INFO_FILE" recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1 @@ -1449,6 +1445,13 @@ else # joiner "$DATA" -mindepth 1 -prune -regex "$cpat" \ -o -exec rm -rf {} >&2 \+ + # Deleting files from previous SST and legacy files from old versions: + [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" + [ -f "$DATA/xtrabackup_checkpoints" ] && rm -f "$DATA/xtrabackup_checkpoints" + [ -f "$DATA/xtrabackup_info" ] && rm -f "$DATA/xtrabackup_info" + [ -f "$DATA/xtrabackup_slave_info" ] && rm -f "$DATA/xtrabackup_slave_info" + [ -f "$DATA/xtrabackup_binlog_pos_innodb" ] && rm -f "$DATA/xtrabackup_binlog_pos_innodb" + TDATA="$DATA" DATA="$DATA/.sst" MAGIC_FILE="$DATA/$INFO_FILE" @@ -1562,6 +1565,7 @@ else # joiner fi MAGIC_FILE="$TDATA/$INFO_FILE" + DONOR_MAGIC_FILE="$TDATA/$DONOR_INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" timeit 'mariadb-backup move stage' "$INNOMOVE" @@ -1586,7 +1590,8 @@ else # joiner fi if [ ! -r "$MAGIC_FILE" ]; then - wsrep_log_error "SST magic file '$MAGIC_FILE' not found/readable" + wsrep_log_error "Internal error: SST magic file '$MAGIC_FILE'" \ + "not found or not readable" exit 2 fi From fbd882914928c3cd06823993f031403d54c6c8c0 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 10 Sep 2024 03:45:19 +0200 Subject: [PATCH 07/17] galera SST scripts: removing obsolete xtrabackup_pid support Removed handling of the long-unsupported xtrabackup_pid file, as it is not even created by modern versions of mariabackup. Instead, added stopping of the asynchronous process that mariabackup runs (if it is still active) to the exception handler. --- scripts/wsrep_sst_mariabackup.sh | 31 +++++++++++++++---------------- scripts/wsrep_sst_rsync.sh | 13 ++++++------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index de219283d15..c607cb7d7eb 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -687,16 +687,16 @@ cleanup_at_exit() fi if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + if [ -n "$BACKUP_PID" ]; then + if ps -p $BACKUP_PID >/dev/null 2>&1; then + wsrep_log_error \ + "mariadb-backup process is still running. Killing..." + cleanup_pid $CHECK_PID + fi + fi wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file else - if [ -n "$BACKUP_PID" ]; then - if check_pid "$BACKUP_PID" 1; then - wsrep_log_error \ - "mariadb-backup process is still running. Killing..." - cleanup_pid $CHECK_PID "$BACKUP_PID" - fi - fi [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" || : fi @@ -906,14 +906,14 @@ monitor_process() local sst_stream_pid=$1 while :; do - if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then + if ! ps -p $WSREP_SST_OPT_PARENT >/dev/null 2>&1; then wsrep_log_error \ "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ "terminated unexpectedly." - kill -- -"$WSREP_SST_OPT_PARENT" + kill -- -$WSREP_SST_OPT_PARENT exit 32 fi - if ! ps -p "$sst_stream_pid" >/dev/null 2>&1; then + if ! ps -p $sst_stream_pid >/dev/null 2>&1; then break fi sleep 0.1 @@ -1073,7 +1073,7 @@ SST_PID="$DATA/wsrep_sst.pid" # give some time for previous SST to complete: check_round=0 -while check_pid "$SST_PID" 0; do +while check_pid "$SST_PID"; do wsrep_log_info "previous SST is not completed, waiting for it to exit" check_round=$(( check_round+1 )) if [ $check_round -eq 30 ]; then @@ -1220,9 +1220,6 @@ if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then exit 22 fi - # mariadb-backup implicitly writes PID to fixed location in $xtmpdir - BACKUP_PID="$xtmpdir/xtrabackup_pid" - else # BYPASS FOR IST wsrep_log_info "Bypassing the SST for IST" @@ -1402,7 +1399,7 @@ else # joiner fi mkdir -p "$DATA/.sst" (recv_joiner "$DATA/.sst" "$stagemsg-SST" 0 0 0) & - jpid=$! + BACKUP_PID=$! wsrep_log_info "Proceeding with SST" get_binlog @@ -1447,6 +1444,7 @@ else # joiner # Deleting files from previous SST and legacy files from old versions: [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" + [ -f "$DATA/xtrabackup_pid" ] && rm -f "$DATA/xtrabackup_pid" [ -f "$DATA/xtrabackup_checkpoints" ] && rm -f "$DATA/xtrabackup_checkpoints" [ -f "$DATA/xtrabackup_info" ] && rm -f "$DATA/xtrabackup_info" [ -f "$DATA/xtrabackup_slave_info" ] && rm -f "$DATA/xtrabackup_slave_info" @@ -1457,7 +1455,8 @@ else # joiner MAGIC_FILE="$DATA/$INFO_FILE" wsrep_log_info "Waiting for SST streaming to complete!" - monitor_process $jpid + monitor_process $BACKUP_PID + BACKUP_PID="" if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then wsrep_log_error "xtrabackup_checkpoints missing," \ diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index b05f4bc2efd..55b94ff6009 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -351,7 +351,7 @@ SST_PID="$DATA/wsrep_sst.pid" # give some time for previous SST to complete: check_round=0 -while check_pid "$SST_PID" 0; do +while check_pid "$SST_PID"; do wsrep_log_info "Previous SST is not completed, waiting for it to exit" check_round=$(( check_round+1 )) if [ $check_round -eq 20 ]; then @@ -866,19 +866,18 @@ EOF echo "ready $ADDR:$RSYNC_PORT/$MODULE" - MYSQLD_PID="$WSREP_SST_OPT_PARENT" - # wait for SST to complete by monitoring magic file while [ ! -r "$MAGIC_FILE" ] && check_pid "$TRANSFER_PID" && \ - ps -p $MYSQLD_PID >/dev/null 2>&1 + ps -p $WSREP_SST_OPT_PARENT >/dev/null 2>&1 do sleep 1 done - if ! ps -p $MYSQLD_PID >/dev/null 2>&1; then + if ! ps -p $WSREP_SST_OPT_PARENT >/dev/null 2>&1; then wsrep_log_error \ - "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly." - kill -- -$MYSQLD_PID + "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ + "terminated unexpectedly." + kill -- -$WSREP_SST_OPT_PARENT sleep 1 exit 32 fi From 4cb73f49bc2b247e3f3674940b861989a3c9302a Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 10 Sep 2024 22:57:51 +0200 Subject: [PATCH 08/17] galera SST scripts: unification of wsrep_sst_backup with the other scripts --- scripts/wsrep_sst_backup.sh | 42 +++++++++++++------------------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/scripts/wsrep_sst_backup.sh b/scripts/wsrep_sst_backup.sh index a28dcf494b2..6cf17fe7228 100644 --- a/scripts/wsrep_sst_backup.sh +++ b/scripts/wsrep_sst_backup.sh @@ -21,50 +21,32 @@ set -ue # This is a reference script for rsync-based state snapshot transfer -RSYNC_REAL_PID=0 # rsync process id -STUNNEL_REAL_PID=0 # stunnel process id - OS="$(uname)" [ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH -# Setting the path for lsof on CentOS -export PATH="/usr/sbin:/sbin:$PATH" - . $(dirname "$0")/wsrep_sst_common +wsrep_check_datadir -MAGIC_FILE="$WSREP_SST_OPT_DATA/backup_sst_complete" -rm -r "$MAGIC_FILE" +DATA="$WSREP_SST_OPT_DATA" -WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} -# if WSREP_LOG_DIR env. variable is not set, try to get it from my.cnf -if [ -z "$WSREP_LOG_DIR" ]; then - WSREP_LOG_DIR=$(parse_cnf mysqld innodb-log-group-home-dir '') -fi +MAGIC_FILE="$DATA/backup_sst_complete" -if [ -n "$WSREP_LOG_DIR" ]; then - # handle both relative and absolute paths - WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$WSREP_LOG_DIR"; cd $WSREP_LOG_DIR; pwd -P) -else - # default to datadir - WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P) -fi +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] then - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - RC=0 if [ $WSREP_SST_OPT_BYPASS -eq 0 ]; then - FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed" - ERROR="$WSREP_SST_OPT_DATA/sst_error" + FLUSHED="$DATA/tables_flushed" + ERROR="$DATA/sst_error" [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo "flush tables" + echo 'flush tables' # Wait for : # (a) Tables to be flushed, AND @@ -77,19 +59,23 @@ then # Check whether ERROR file exists. if [ -f "$ERROR" ]; then # Flush tables operation failed. - rm -f "$ERROR" + rm "$ERROR" exit 255 fi sleep 0.2 done STATE=$(cat "$FLUSHED") - rm -f "$FLUSHED" - + rm "$FLUSHED" else # BYPASS wsrep_log_info "Bypassing state dump." + + # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id + # (separated by a space). + STATE="$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" + fi echo 'continue' # now server can resume updating data From 606c867e7fec4507bd1225f9abf2d0aa664b6ab6 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Wed, 11 Sep 2024 18:53:24 +0200 Subject: [PATCH 09/17] galera SST scripts: moving common code to wsrep_sst_common file --- scripts/wsrep_sst_backup.sh | 5 +- scripts/wsrep_sst_common.sh | 102 +++++++++++++++- scripts/wsrep_sst_mariabackup.sh | 89 +------------- scripts/wsrep_sst_rsync.sh | 198 +++++++++++-------------------- 4 files changed, 172 insertions(+), 222 deletions(-) diff --git a/scripts/wsrep_sst_backup.sh b/scripts/wsrep_sst_backup.sh index 6cf17fe7228..af2e396641b 100644 --- a/scripts/wsrep_sst_backup.sh +++ b/scripts/wsrep_sst_backup.sh @@ -21,14 +21,13 @@ set -ue # This is a reference script for rsync-based state snapshot transfer -OS="$(uname)" -[ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH - . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir DATA="$WSREP_SST_OPT_DATA" +create_data + MAGIC_FILE="$DATA/backup_sst_complete" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index b6b8c0501c8..4766d82181d 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -20,8 +20,13 @@ trap 'exit 32' HUP PIPE trap 'exit 3' INT QUIT TERM -# Setting the path for some utilities on CentOS -export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" +OS="$(uname)" + +# Setting the paths for some utilities on CentOS +export PATH="${PATH:+$PATH:}/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin" +if [ "$OS" != 'Darwin' ]; then + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}/usr/local/lib:/usr/lib:/lib:/opt/lib" +fi commandex() { @@ -1724,4 +1729,97 @@ simple_cleanup() exit $estatus } +create_data() +{ + OLD_PWD="$(pwd)" + + if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" + fi + DATA_DIR="$(pwd)" + + cd "$OLD_PWD" +} + +create_dirs() +{ + local simplify=${1:-0} + + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' -a \ + "$INNODB_DATA_HOME_DIR" != "$DATA_DIR" ] + then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" + [ $simplify -ne 0 -a "$ib_home_dir" = "$DATA_DIR" ] && ib_home_dir="" + fi + + # if no command line argument and INNODB_LOG_GROUP_HOME is not set, + # then try to get it from the my.cnf: + if [ -z "$INNODB_LOG_GROUP_HOME" ]; then + INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") + fi + + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' -a \ + "$INNODB_LOG_GROUP_HOME" != "$DATA_DIR" ] + then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" + [ $simplify -ne 0 -a "$ib_log_dir" = "$DATA_DIR" ] && ib_log_dir="" + fi + + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: + if [ -z "$INNODB_UNDO_DIR" ]; then + INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") + fi + + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' -a \ + "$INNODB_UNDO_DIR" != "$DATA_DIR" ] + then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" + [ $simplify -ne 0 -a "$ib_undo_dir" = "$DATA_DIR" ] && ib_undo_dir="" + fi + + # if no command line argument then try to get it from the my.cnf: + if [ -z "$ARIA_LOG_DIR" ]; then + ARIA_LOG_DIR=$(parse_cnf '--mysqld' 'aria-log-dir-path') + ARIA_LOG_DIR=$(trim_dir "$ARIA_LOG_DIR") + fi + + if [ -n "$ARIA_LOG_DIR" -a "$ARIA_LOG_DIR" != '.' -a \ + "$ARIA_LOG_DIR" != "$DATA_DIR" ] + then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$ARIA_LOG_DIR" ] && mkdir -p "$ARIA_LOG_DIR" + cd "$ARIA_LOG_DIR" + ar_log_dir="$(pwd)" + cd "$OLD_PWD" + [ $simplify -ne 0 -a "$ar_log_dir" = "$DATA_DIR" ] && ar_log_dir="" + fi +} + wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE started on $WSREP_SST_OPT_ROLE" diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index c607cb7d7eb..75c1e846c76 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -23,8 +23,6 @@ set -ue # https://mariadb.com/kb/en/mariabackup-overview/ # Make sure to read that before proceeding! -OS="$(uname)" - . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir @@ -935,15 +933,7 @@ if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -OLD_PWD="$(pwd)" - -if [ -n "$DATA" -a "$DATA" != '.' ]; then - [ ! -d "$DATA" ] && mkdir -p "$DATA" - cd "$DATA" -fi -DATA_DIR="$(pwd)" - -cd "$OLD_PWD" +create_data if [ $ssyslog -eq 1 ]; then if [ -n "$(commandex logger)" ]; then @@ -1249,85 +1239,12 @@ if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then else # joiner + create_dirs 1 + [ -e "$SST_PROGRESS_FILE" ] && \ wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - # if no command line argument and INNODB_DATA_HOME_DIR environment - # variable is not set, try to get it from the my.cnf: - if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') - INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") - fi - - if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' -a \ - "$INNODB_DATA_HOME_DIR" != "$DATA_DIR" ] - then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" - ib_home_dir="$(pwd)" - cd "$OLD_PWD" - [ "$ib_home_dir" = "$DATA_DIR" ] && ib_home_dir="" - fi - - # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # then try to get it from the my.cnf: - if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') - INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") - fi - - if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' -a \ - "$INNODB_LOG_GROUP_HOME" != "$DATA_DIR" ] - then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" - cd "$INNODB_LOG_GROUP_HOME" - ib_log_dir="$(pwd)" - cd "$OLD_PWD" - [ "$ib_log_dir" = "$DATA_DIR" ] && ib_log_dir="" - fi - - # if no command line argument and INNODB_UNDO_DIR is not set, - # then try to get it from the my.cnf: - if [ -z "$INNODB_UNDO_DIR" ]; then - INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') - INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") - fi - - if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' -a \ - "$INNODB_UNDO_DIR" != "$DATA_DIR" ] - then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" - cd "$INNODB_UNDO_DIR" - ib_undo_dir="$(pwd)" - cd "$OLD_PWD" - [ "$ib_undo_dir" = "$DATA_DIR" ] && ib_undo_dir="" - fi - - # if no command line argument then try to get it from the my.cnf: - if [ -z "$ARIA_LOG_DIR" ]; then - ARIA_LOG_DIR=$(parse_cnf '--mysqld' 'aria-log-dir-path') - ARIA_LOG_DIR=$(trim_dir "$ARIA_LOG_DIR") - fi - - if [ -n "$ARIA_LOG_DIR" -a "$ARIA_LOG_DIR" != '.' -a \ - "$ARIA_LOG_DIR" != "$DATA_DIR" ] - then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$ARIA_LOG_DIR" ] && mkdir -p "$ARIA_LOG_DIR" - cd "$ARIA_LOG_DIR" - ar_log_dir="$(pwd)" - cd "$OLD_PWD" - [ "$ar_log_dir" = "$DATA_DIR" ] && ar_log_dir="" - fi - if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" fi diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 55b94ff6009..c22d0b3e494 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -21,17 +21,14 @@ set -ue # This is a reference script for rsync-based state snapshot transfer -RSYNC_REAL_PID=0 # rsync process id -STUNNEL_REAL_PID=0 # stunnel process id - -OS="$(uname)" -[ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH - . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir wsrep_check_programs rsync +RSYNC_REAL_PID=0 # rsync process id +STUNNEL_REAL_PID=0 # stunnel process id + cleanup_joiner() { # Since this is invoked just after exit NNN @@ -162,15 +159,7 @@ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi -OLD_PWD="$(pwd)" - -if [ -n "$DATA" -a "$DATA" != '.' ]; then - [ ! -d "$DATA" ] && mkdir -p "$DATA" - cd "$DATA" -fi -DATA_DIR="$(pwd)" - -cd "$OLD_PWD" +create_data BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" @@ -179,77 +168,6 @@ ib_log_dir="$DATA_DIR" ib_home_dir="$DATA_DIR" ib_undo_dir="$DATA_DIR" -# if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# then try to get it from the my.cnf: -if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') - INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") -fi - -if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' -a \ - "$INNODB_LOG_GROUP_HOME" != "$DATA_DIR" ] -then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" - cd "$INNODB_LOG_GROUP_HOME" - ib_log_dir="$(pwd)" - cd "$OLD_PWD" -fi - -# if no command line argument and INNODB_DATA_HOME_DIR environment -# variable is not set, try to get it from the my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') - INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") -fi - -if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' -a \ - "$INNODB_DATA_HOME_DIR" != "$DATA_DIR" ] -then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" - ib_home_dir="$(pwd)" - cd "$OLD_PWD" -fi - -# if no command line argument and INNODB_UNDO_DIR is not set, -# then try to get it from the my.cnf: -if [ -z "$INNODB_UNDO_DIR" ]; then - INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') - INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") -fi - -if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' -a \ - "$INNODB_UNDO_DIR" != "$DATA_DIR" ] -then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" - cd "$INNODB_UNDO_DIR" - ib_undo_dir="$(pwd)" - cd "$OLD_PWD" -fi - -# if no command line argument then try to get it from the my.cnf: -if [ -z "$ARIA_LOG_DIR" ]; then - ARIA_LOG_DIR=$(parse_cnf '--mysqld' 'aria-log-dir-path') - ARIA_LOG_DIR=$(trim_dir "$ARIA_LOG_DIR") -fi - -if [ -n "$ARIA_LOG_DIR" -a "$ARIA_LOG_DIR" != '.' -a \ - "$ARIA_LOG_DIR" != "$DATA_DIR" ] -then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$ARIA_LOG_DIR" ] && mkdir -p "$ARIA_LOG_DIR" - cd "$ARIA_LOG_DIR" - ar_log_dir="$(pwd)" - cd "$OLD_PWD" -fi - encgroups='--mysqld|sst' check_server_ssl_config @@ -598,66 +516,82 @@ FILTER="-f '- /lost+found' wsrep_log_info "Transfer of normal directories done" - # Transfer InnoDB data files - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --dirs --delete --quiet \ - $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$ib_home_dir/" \ - "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? + if [ -d "$ib_home_dir" ]; then + + # Transfer InnoDB data files + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ + -f '- **' "$ib_home_dir/" \ + "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_data_home_dir returned code $RC:" + exit 255 # unknown error + fi + + wsrep_log_info "Transfer of InnoDB data files done" - if [ $RC -ne 0 ]; then - wsrep_log_error "rsync innodb_data_home_dir returned code $RC:" - exit 255 # unknown error fi - wsrep_log_info "Transfer of InnoDB data files done" + if [ -d "$ib_log_dir" ]; then - # second, we transfer InnoDB log files - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --dirs --delete --quiet \ - $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' \ - -f '- **' "$ib_log_dir/" \ - "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? + # second, we transfer InnoDB log files + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' \ + -f '- **' "$ib_log_dir/" \ + "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_log_group_home_dir returned code $RC:" + exit 255 # unknown error + fi + + wsrep_log_info "Transfer of InnoDB log files done" - if [ $RC -ne 0 ]; then - wsrep_log_error "rsync innodb_log_group_home_dir returned code $RC:" - exit 255 # unknown error fi - wsrep_log_info "Transfer of InnoDB log files done" + if [ "$ib_undo_dir" ]; then - # third, we transfer InnoDB undo logs - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --dirs --delete --quiet \ - $WHOLE_FILE_OPT -f '+ /undo*' \ - -f '- **' "$ib_undo_dir/" \ - "rsync://$WSREP_SST_OPT_ADDR-undo_dir" >&2 || RC=$? + # third, we transfer InnoDB undo logs + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /undo*' \ + -f '- **' "$ib_undo_dir/" \ + "rsync://$WSREP_SST_OPT_ADDR-undo_dir" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_undo_dir returned code $RC:" + exit 255 # unknown error + fi + + wsrep_log_info "Transfer of InnoDB undo logs done" - if [ $RC -ne 0 ]; then - wsrep_log_error "rsync innodb_undo_dir returned code $RC:" - exit 255 # unknown error fi - wsrep_log_info "Transfer of InnoDB undo logs done" + if [ "$ar_log_dir" ]; then - # fourth, we transfer Aria logs - rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --dirs --delete --quiet \ - $WHOLE_FILE_OPT -f '+ /aria_log_control' -f '+ /aria_log.*' \ - -f '- **' "$ar_log_dir/" \ - "rsync://$WSREP_SST_OPT_ADDR-aria_log" >&2 || RC=$? + # fourth, we transfer Aria logs + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ + --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /aria_log_control' -f '+ /aria_log.*' \ + -f '- **' "$ar_log_dir/" \ + "rsync://$WSREP_SST_OPT_ADDR-aria_log" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync aria_log_dir_path returned code $RC:" + exit 255 # unknown error + fi + + wsrep_log_info "Transfer of Aria logs done" - if [ $RC -ne 0 ]; then - wsrep_log_error "rsync aria_log_dir_path returned code $RC:" - exit 255 # unknown error fi - wsrep_log_info "Transfer of Aria logs done" - # then, we parallelize the transfer of database directories, # use '.' so that path concatenation works: @@ -744,6 +678,8 @@ else # joiner check_sockets_utils + create_dirs + ADDR="$WSREP_SST_OPT_HOST" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST" From 202fd502cf31f5120c5361641a47e8cd4830984c Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 12 Sep 2024 17:29:24 +0200 Subject: [PATCH 10/17] galera SST scripts: fixes for error logging in non-linux systems --- scripts/wsrep_sst_common.sh | 59 ++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 4766d82181d..581f804d2d3 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -39,11 +39,45 @@ commandex() fi } +with_bash_42=0 with_printf=1 if [ -z "$BASH_VERSION" ]; then [ -z "$(commandex printf)" ] && with_printf=0 +else + [ "${BASH_VERSINFO[0]}" -eq 4 -a "${BASH_VERSINFO[1]}" -ge 2 -o \ + "${BASH_VERSINFO[0]}" -gt 4 ] && with_bash_42=1 fi +wsrep_log() +{ + local t + # echo everything to stderr so that it gets into common error log + # deliberately made to look different from the rest of the log + if [ "$OS" = 'Linux' ]; then + t=$(date '+%Y%m%d %H:%M:%S.%3N') + elif [ $with_bash_42 -ne 0 ]; then + printf -v t '%(%Y%m%d %H:%M:%S)T.000' + else + t=$(date '+%Y%m%d %H:%M:%S.000') + fi + echo "WSREP_SST: $* ($t)" >&2 +} + +wsrep_log_error() +{ + wsrep_log "[ERROR] $*" +} + +wsrep_log_warning() +{ + wsrep_log "[WARNING] $*" +} + +wsrep_log_info() +{ + wsrep_log "[INFO] $*" +} + trim_string() { if [ -n "$BASH_VERSION" ]; then @@ -844,29 +878,6 @@ else MYSQLDUMP=$(commandex 'mysqldump') fi -wsrep_log() -{ - # echo everything to stderr so that it gets into common error log - # deliberately made to look different from the rest of the log - local readonly tst=$(date "+%Y%m%d %H:%M:%S.%N" | cut -b -21) - echo "WSREP_SST: $* ($tst)" >&2 -} - -wsrep_log_error() -{ - wsrep_log "[ERROR] $*" -} - -wsrep_log_warning() -{ - wsrep_log "[WARNING] $*" -} - -wsrep_log_info() -{ - wsrep_log "[INFO] $*" -} - if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then MY_PRINT_DEFAULTS="$SCRIPTS_DIR/my_print_defaults" elif [ -x "$EXTRA_DIR/my_print_defaults" ]; then @@ -1090,7 +1101,7 @@ wsrep_check_program() local prog="$1" local cmd=$(commandex "$prog") if [ -z "$cmd" ]; then - echo "'$prog' not found in PATH" + wsrep_log_error "'$prog' not found in path" return 2 # no such file or directory fi } From 642195d255619854d13f20b9937b604dbbe2a5fb Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 12 Sep 2024 19:00:26 +0200 Subject: [PATCH 11/17] MDEV-34234: SST hangs when running on unprivileged containers on RHEL9 The lsof utility is prone to blocking on system calls that it uses to obtain information about sockets (or files, devices, etc.). This behavior is described in its own documentation. It has a '-b' option (in combination with warnings suppression via '-w') that reduces the probability of blocking, introducing new problems (luckily probably not relevant for our use case). However, there is no guarantee that it will not hang on some distributions, with some TCP/IP stack implementations, or with some filesystems, etc. Also, of the three utilities that are suitable for our purposes, lsof is the slowest. So if there are other utilities that we use during SST, such as 'ss' or 'sockstat', it is reasonable to use them instead of lsof. This commit changes the prioritization of utilities, it does not need additional tests (besides the numerous SST tests already available in the galera suites). If the system still need to use lsof, this commit adds the '-b' and '-w' options to it command line - to reduce the likelihood of blocking. --- scripts/wsrep_sst_common.sh | 40 +++++++++++++++++++++++-------------- scripts/wsrep_sst_rsync.sh | 32 +++++++++++++++++------------ 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 581f804d2d3..43732d7b49f 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1224,14 +1224,18 @@ check_sockets_utils() sockstat_available=0 ss_available=0 - [ -n "$(commandex lsof)" ] && lsof_available=1 - [ -n "$(commandex sockstat)" ] && sockstat_available=1 - [ -n "$(commandex ss)" ] && ss_available=1 - - if [ $lsof_available -eq 0 -a \ - $sockstat_available -eq 0 -a \ - $ss_available -eq 0 ] - then + # The presence of any of these utilities is enough for us: + if [ -n "$(commandex ss)" ]; then + ss_available=1 + elif [ -n "$(commandex sockstat)" ]; then + sockstat_available=1 + elif [ -n "$(commandex lsof)" ]; then + lsof_available=1 + # Let's check that lsof has an option to bypass blocking: + if lsof -h 2>&1 | grep -qw -F -- '-b'; then + lsof_available=2 + fi + else wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in" \ "the PATH. Make sure you have it installed." exit 2 # ENOENT @@ -1258,9 +1262,9 @@ check_port() local rc=1 - if [ $lsof_available -ne 0 ]; then - lsof -Pnl -i ":$port" 2>/dev/null | \ - grep -q -E "^($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*\\(LISTEN\\)" && rc=0 + if [ $ss_available -ne 0 ]; then + ss -nlpH "( sport = :$port )" 2>/dev/null | \ + grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 elif [ $sockstat_available -ne 0 ]; then local opts='-p' if [ "$OS" = 'FreeBSD' ]; then @@ -1268,11 +1272,17 @@ check_port() # to display the connection state: opts='-sp' fi - sockstat "$opts" "$port" 2>/dev/null | \ + sockstat $opts "$port" 2>/dev/null | \ grep -q -E "[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*[[:space:]]LISTEN" && rc=0 - elif [ $ss_available -ne 0 ]; then - ss -nlpH "( sport = :$port )" 2>/dev/null | \ - grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 + elif [ $lsof_available -ne 0 ]; then + local lsof_opts='-Pnl' + if [ $lsof_available -gt 1 ]; then + lsof_opts="$lsof_opts -b -w" + else + lsof_opts="$lsof_opts -S 15" + fi + lsof $lsof_opts -i ":$port" 2>/dev/null | \ + grep -q -E "^($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*\\(LISTEN\\)" && rc=0 else wsrep_log_error "Unknown sockets utility" exit 2 # ENOENT diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index c22d0b3e494..c5640d2a22c 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -96,29 +96,35 @@ check_pid_and_port() local port_info local busy=0 - if [ $lsof_available -ne 0 ]; then - port_info=$(lsof -Pnl -i ":$port" 2>/dev/null | grep -F '(LISTEN)') - echo "$port_info" | \ - grep -q -E "[[:space:]]\\[?(\\*|[[:xdigit:]]*(:[[:xdigit:]]*)+)(\\](%[^:]+)?)?:$port[[:space:]]" && busy=1 - else - local filter='([^[:space:]]+[[:space:]]+){4}[^[:space:]]+' - if [ $sockstat_available -ne 0 ]; then + if [ $ss_available -ne 0 -o $sockstat_available -ne 0 ]; then + if [ $ss_available -ne 0 ]; then + port_info=$(ss -nlpH "( sport = :$port )" 2>/dev/null | \ + grep -F 'users:(' | grep -o -E "([^[:space:]]+[[:space:]]+){4}[^[:space:]]+") + else local opts='-p' + local terms=4 if [ "$OS" = 'FreeBSD' ]; then # sockstat on FreeBSD requires the "-s" option # to display the connection state: opts='-sp' # in addition, sockstat produces an additional column: - filter='([^[:space:]]+[[:space:]]+){5}[^[:space:]]+' + terms=5 fi - port_info=$(sockstat "$opts" "$port" 2>/dev/null | \ - grep -E '[[:space:]]LISTEN' | grep -o -E "$filter") - else - port_info=$(ss -nlpH "( sport = :$port )" 2>/dev/null | \ - grep -F 'users:(' | grep -o -E "$filter") + port_info=$(sockstat $opts "$port" 2>/dev/null | \ + grep -E '[[:space:]]LISTEN' | grep -o -E "([^[:space:]]+[[:space:]]+){$terms}[^[:space:]]+") fi echo "$port_info" | \ grep -q -E "[[:space:]]\\[?(\\*|[[:xdigit:]]*(:[[:xdigit:]]*)+)(\\](%[^:]+)?)?:$port\$" && busy=1 + else + local lsof_opts='-Pnl' + if [ $lsof_available -gt 1 ]; then + lsof_opts="$lsof_opts -b -w" + else + lsof_opts="$lsof_opts -S 15" + fi + port_info=$(lsof $lsof_opts -i ":$port" 2>/dev/null | grep -F '(LISTEN)' || :) + echo "$port_info" | \ + grep -q -E "[[:space:]]\\[?(\\*|[[:xdigit:]]*(:[[:xdigit:]]*)+)(\\](%[^:]+)?)?:$port[[:space:]]" && busy=1 fi if [ $busy -eq 0 ]; then From 7742cc9ff98ef7253c355e19b52d9f06287e0ee7 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Fri, 13 Sep 2024 16:46:12 +0200 Subject: [PATCH 12/17] galera SST scripts: more robust port checking --- scripts/wsrep_sst_common.sh | 86 +++++++++++++++++++++----------- scripts/wsrep_sst_mariabackup.sh | 2 +- scripts/wsrep_sst_rsync.sh | 73 +++++++++++++-------------- 3 files changed, 93 insertions(+), 68 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 43732d7b49f..148aa87f791 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1220,26 +1220,55 @@ is_local_ip() check_sockets_utils() { + # The presence of any of these utilities is enough for us: lsof_available=0 sockstat_available=0 ss_available=0 - # The presence of any of these utilities is enough for us: - if [ -n "$(commandex ss)" ]; then + socket_utility="$(commandex ss)" + if [ -n "$socket_utility" ]; then + socket_uname='ss' ss_available=1 - elif [ -n "$(commandex sockstat)" ]; then - sockstat_available=1 - elif [ -n "$(commandex lsof)" ]; then - lsof_available=1 - # Let's check that lsof has an option to bypass blocking: - if lsof -h 2>&1 | grep -qw -F -- '-b'; then - lsof_available=2 + ss_opts='-nlp' + # Let's check that ss has an option to skip headers: + if $socket_utility -h 2>&1 | grep -qw -F -- '-H'; then + ss_available=2 + ss_opts="${ss_opts}H" fi else - wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in" \ - "the PATH. Make sure you have it installed." - exit 2 # ENOENT + socket_utility="$(commandex sockstat)" + if [ -n "$socket_utility" ]; then + socket_uname='sockstat' + sockstat_available=1 + sockstat_opts='-p' + if [ "$OS" = 'FreeBSD' ]; then + # sockstat in FreeBSD is different from other systems, + # let's denote it with a different value: + sockstat_available=2 + fi + else + socket_utility="$(commandex lsof)" + if [ -n "$socket_utility" ]; then + socket_uname='lsof' + lsof_available=1 + lsof_opts='-Pnl' + # Let's check that lsof has an option to bypass blocking: + if $socket_utility -h 2>&1 | grep -qw -F -- '-b'; then + lsof_available=2 + lsof_opts="$lsof_opts -b -w" + else + lsof_opts="$lsof_opts -S 10" + fi + else + wsrep_log_error "Neither lsof, nor sockstat, nor ss tool" \ + "were found in the path. Make sure you have" \ + "at least one of them installed." + exit 2 # ENOENT + fi + fi fi + wsrep_log_info "'$socket_uname' is selected as a socket" \ + "information utility." } # @@ -1263,26 +1292,23 @@ check_port() local rc=1 if [ $ss_available -ne 0 ]; then - ss -nlpH "( sport = :$port )" 2>/dev/null | \ - grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 + $socket_utility $ss_opts -t "( sport = :$port )" 2>/dev/null | \ + grep -q -E "[[:space:]]users:[[:space:]]?\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 elif [ $sockstat_available -ne 0 ]; then - local opts='-p' - if [ "$OS" = 'FreeBSD' ]; then - # sockstat on FreeBSD requires the "-s" option - # to display the connection state: - opts='-sp' - fi - sockstat $opts "$port" 2>/dev/null | \ - grep -q -E "[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*[[:space:]]LISTEN" && rc=0 - elif [ $lsof_available -ne 0 ]; then - local lsof_opts='-Pnl' - if [ $lsof_available -gt 1 ]; then - lsof_opts="$lsof_opts -b -w" + if [ $sockstat_available -gt 1 ]; then + # sockstat on FreeBSD does not return the connection + # state without special option that cancel filtering + # by the port, so we ignore the connection state for + # this system: + $socket_utility $sockstat_opts "$port" 2>/dev/null | \ + grep -q -E "^[^[:space:]]+[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid([[:space:]]|\$)" && rc=0 else - lsof_opts="$lsof_opts -S 15" + $socket_utility $sockstat_opts "$port" 2>/dev/null | \ + grep -q -E "^[^[:space:]]+[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid([[:space:]].+)?[[:space:]]LISTEN([[:space:]]|\$)" && rc=0 fi - lsof $lsof_opts -i ":$port" 2>/dev/null | \ - grep -q -E "^($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*\\(LISTEN\\)" && rc=0 + elif [ $lsof_available -ne 0 ]; then + $socket_utility $lsof_opts -i ":$port" 2>/dev/null | \ + grep -q -E "^($utils)[^[:space:]]*[[:space:]]+$pid([[:space:]].+)?[[:space:]]\\(LISTEN\\)([[:space:]]|\$)" && rc=0 else wsrep_log_error "Unknown sockets utility" exit 2 # ENOENT @@ -1553,7 +1579,7 @@ get_proc() if [ -z "$nproc" ]; then set +e if [ "$OS" = 'Linux' ]; then - nproc=$(grep -cw -E '^processor' /proc/cpuinfo 2>/dev/null) + nproc=$(grep -cw -E '^processor' /proc/cpuinfo 2>/dev/null || :) elif [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ]; then nproc=$(sysctl -n hw.ncpu) fi diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 75c1e846c76..7f6de68eba9 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -751,7 +751,7 @@ setup_ports() wait_for_listen() { for i in {1..150}; do - if check_port "" "$SST_PORT" 'socat|nc'; then + if check_port "" "$SST_PORT" 'socat|nc|netcat'; then break fi sleep 0.2 diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index c5640d2a22c..0ebe8063823 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -92,53 +92,52 @@ check_pid_and_port() local utils='rsync|stunnel' - if ! check_port $pid "$port" "$utils"; then - local port_info - local busy=0 + local port_info + local final + if ! check_port $pid "$port" "$utils"; then if [ $ss_available -ne 0 -o $sockstat_available -ne 0 ]; then if [ $ss_available -ne 0 ]; then - port_info=$(ss -nlpH "( sport = :$port )" 2>/dev/null | \ - grep -F 'users:(' | grep -o -E "([^[:space:]]+[[:space:]]+){4}[^[:space:]]+") + port_info=$($socket_utility $ss_opts -t "( sport = :$port )" 2>/dev/null | \ + grep -E '[[:space:]]users:[[:space:]]?(' | \ + grep -o -E "([^[:space:]]+[[:space:]]+){4}[^[:space:]]+" || :) else - local opts='-p' - local terms=4 - if [ "$OS" = 'FreeBSD' ]; then - # sockstat on FreeBSD requires the "-s" option - # to display the connection state: - opts='-sp' - # in addition, sockstat produces an additional column: - terms=5 + if [ $sockstat_available -gt 1 ]; then + # sockstat on FreeBSD does not return the connection + # state without special option that cancel filtering + # by the port, so we ignore the connection state for + # this system, also on the FreeBSD sockstat utility + # produces an additional column: + port_info=$($socket_utility $sockstat_opts "$port" 2>/dev/null | \ + grep -o -E "([^[:space:]]+[[:space:]]+){5}[^[:space:]]+" || :) + else + port_info=$($socket_utility $sockstat_opts "$port" 2>/dev/null | \ + grep -E '[[:space:]]LISTEN([[:space:]]|$)' | \ + grep -o -E "([^[:space:]]+[[:space:]]+){4}[^[:space:]]+" || :) fi - port_info=$(sockstat $opts "$port" 2>/dev/null | \ - grep -E '[[:space:]]LISTEN' | grep -o -E "([^[:space:]]+[[:space:]]+){$terms}[^[:space:]]+") fi - echo "$port_info" | \ - grep -q -E "[[:space:]]\\[?(\\*|[[:xdigit:]]*(:[[:xdigit:]]*)+)(\\](%[^:]+)?)?:$port\$" && busy=1 + final='$' else - local lsof_opts='-Pnl' - if [ $lsof_available -gt 1 ]; then - lsof_opts="$lsof_opts -b -w" - else - lsof_opts="$lsof_opts -S 15" - fi - port_info=$(lsof $lsof_opts -i ":$port" 2>/dev/null | grep -F '(LISTEN)' || :) - echo "$port_info" | \ - grep -q -E "[[:space:]]\\[?(\\*|[[:xdigit:]]*(:[[:xdigit:]]*)+)(\\](%[^:]+)?)?:$port[[:space:]]" && busy=1 + port_info=$($socket_utility $lsof_opts -i ":$port" 2>/dev/null | \ + grep -w -F '(LISTEN)' || :) + final='[[:space:]]' + fi + + local busy=0 + if [ -n "$port_info" ]; then + local address='(\*|[0-9a-fA-F]*(:[0-9a-fA-F]*){1,7}|[0-9]+(\.[0-9]+){3})' + local filter="[[:space:]]($address|\\[$address\\])(%[^:]+)?:$port$final" + echo "$port_info" | grep -q -E "$filter" && busy=1 fi if [ $busy -eq 0 ]; then - if ! echo "$port_info" | grep -qw -F "[$addr]:$port" && \ - ! echo "$port_info" | grep -qw -F -- "$addr:$port" - then - if ! ps -p $pid >/dev/null 2>&1; then - wsrep_log_error \ - "rsync or stunnel daemon (PID: $pid)" \ - "terminated unexpectedly." - exit 16 # EBUSY - fi - return 1 + if ! ps -p $pid >/dev/null 2>&1; then + wsrep_log_error \ + "the rsync or stunnel daemon (PID: $pid)" \ + "terminated unexpectedly." + exit 16 # EBUSY fi + return 1 fi if ! check_port $pid "$port" "$utils"; then @@ -244,7 +243,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then elif [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then # check if the address is an ip-address (v4 or v6): if echo "$WSREP_SST_OPT_HOST_UNESCAPED" | \ - grep -q -E '^([0-9]+(\.[0-9]+){3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$' + grep -q -E '^([0-9]+(\.[0-9]+){3}|[0-9a-fA-F]*(:[0-9a-fA-F]*){1,7})$' then CHECK_OPT="checkIP = $WSREP_SST_OPT_HOST_UNESCAPED" else From 46a5d2f1cfe9bca9e00731394095f5b79592608d Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Sun, 15 Sep 2024 04:12:27 +0200 Subject: [PATCH 13/17] galera SST scripts: unification of the previous SST completion check --- scripts/wsrep_sst_backup.sh | 7 +-- scripts/wsrep_sst_common.sh | 27 ++++++++++- scripts/wsrep_sst_mariabackup.sh | 79 ++++++++++++-------------------- scripts/wsrep_sst_mysqldump.sh | 2 + scripts/wsrep_sst_rsync.sh | 50 ++++++-------------- 5 files changed, 74 insertions(+), 91 deletions(-) diff --git a/scripts/wsrep_sst_backup.sh b/scripts/wsrep_sst_backup.sh index af2e396641b..e1a8b541cc8 100644 --- a/scripts/wsrep_sst_backup.sh +++ b/scripts/wsrep_sst_backup.sh @@ -22,14 +22,11 @@ set -ue # This is a reference script for rsync-based state snapshot transfer . $(dirname "$0")/wsrep_sst_common -wsrep_check_datadir - -DATA="$WSREP_SST_OPT_DATA" - -create_data MAGIC_FILE="$DATA/backup_sst_complete" +wait_previous_sst + [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 148aa87f791..e6b48c4b6a6 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1769,7 +1769,7 @@ simple_cleanup() if [ $estatus -ne 0 ]; then wsrep_log_error "Cleanup after exit with status: $estatus" fi - if [ -n "${SST_PID:-}" ]; then + if [ -n "$SST_PID" ]; then [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$SST_PID" ] && rm -f "$SST_PID" || : fi @@ -1869,4 +1869,29 @@ create_dirs() fi } +wait_previous_sst() +{ + # give some time for previous SST to complete: + check_round=0 + while check_pid "$SST_PID" 1; do + wsrep_log_info "Previous SST is not completed, waiting for it to exit" + check_round=$(( check_round+1 )) + if [ $check_round -eq 30 ]; then + wsrep_log_error "previous SST script still running..." + exit 114 # EALREADY + fi + sleep 1 + done + + trap simple_cleanup EXIT + echo $$ > "$SST_PID" +} + +DATA="$WSREP_SST_OPT_DATA" + +wsrep_check_datadir +create_data + +SST_PID="$DATA/wsrep_sst.pid" + wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE started on $WSREP_SST_OPT_ROLE" diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 7f6de68eba9..c2d4752f3f2 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -24,7 +24,21 @@ set -ue # Make sure to read that before proceeding! . $(dirname "$0")/wsrep_sst_common -wsrep_check_datadir + +BACKUP_BIN=$(commandex 'mariadb-backup') +if [ -z "$BACKUP_BIN" ]; then + wsrep_log_error 'mariadb-backup binary not found in path' + exit 42 +fi + +BACKUP_PID="" + +INFO_FILE='xtrabackup_galera_info' +DONOR_INFO_FILE='donor_galera_info' +IST_FILE='xtrabackup_ist' + +MAGIC_FILE="$DATA/$INFO_FILE" +DONOR_MAGIC_FILE="$DATA/$DONOR_INFO_FILE" ealgo="" eformat="" @@ -33,7 +47,6 @@ ekeyfile="" encrypt=0 ssyslog="" ssystag="" -BACKUP_PID="" tcert="" tcap="" tpem="" @@ -62,10 +75,10 @@ tcmd="" payload=0 pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p'" pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE" -STATDIR="" uextra=0 disver="" +STATDIR="" tmpopts="" itmpdir="" xtmpdir="" @@ -91,23 +104,6 @@ readonly TOTAL_TAG='total' # For backup locks it is 1 sent by joiner sst_ver=1 -declare -a RC - -BACKUP_BIN=$(commandex 'mariadb-backup') -if [ -z "$BACKUP_BIN" ]; then - wsrep_log_error 'mariadb-backup binary not found in path' - exit 42 -fi - -DATA="$WSREP_SST_OPT_DATA" - -INFO_FILE='xtrabackup_galera_info' -DONOR_INFO_FILE='donor_galera_info' -IST_FILE='xtrabackup_ist' - -MAGIC_FILE="$DATA/$INFO_FILE" -DONOR_MAGIC_FILE="$DATA/$DONOR_INFO_FILE" - INNOAPPLYLOG="$DATA/mariabackup.prepare.log" INNOMOVELOG="$DATA/mariabackup.move.log" INNOBACKUPLOG="$DATA/mariabackup.backup.log" @@ -713,7 +709,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o 'pgid=' $$ 2>/dev/null | grep -o -E '[0-9]+' || :) + local pgid=$(ps -o 'pgid=' $$ 2>/dev/null | grep -o -E '[0-9]+' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -727,7 +723,7 @@ cleanup_at_exit() fi fi - if [ -n "${SST_PID:-}" ]; then + if [ -n "$SST_PID" ]; then [ -f "$SST_PID" ] && rm -f "$SST_PID" || : fi @@ -933,7 +929,17 @@ if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -create_data +get_stream +get_transfer + +findopt='-L' +[ "$OS" = 'FreeBSD' ] && findopt="$findopt -E" + +wait_previous_sst + +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" +[ -f "$DONOR_MAGIC_FILE" ] && rm -f "$DONOR_MAGIC_FILE" +[ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" if [ $ssyslog -eq 1 ]; then if [ -n "$(commandex logger)" ]; then @@ -1053,30 +1059,6 @@ send_magic() fi } -get_stream -get_transfer - -findopt='-L' -[ "$OS" = 'FreeBSD' ] && findopt="$findopt -E" - -SST_PID="$DATA/wsrep_sst.pid" - -# give some time for previous SST to complete: -check_round=0 -while check_pid "$SST_PID"; do - wsrep_log_info "previous SST is not completed, waiting for it to exit" - check_round=$(( check_round+1 )) - if [ $check_round -eq 30 ]; then - wsrep_log_error "previous SST script still running." - exit 114 # EALREADY - fi - sleep 1 -done - -[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -[ -f "$DONOR_MAGIC_FILE" ] && rm -f "$DONOR_MAGIC_FILE" -[ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" - if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then trap cleanup_at_exit EXIT @@ -1249,9 +1231,6 @@ else # joiner impts="--parallel=$backup_threads${impts:+ }$impts" fi - trap simple_cleanup EXIT - echo $$ > "$SST_PID" - stagemsg='Joiner-Recv' MODULE="${WSREP_SST_OPT_MODULE:-xtrabackup_sst}" diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index a06231424b5..88b7d13cf5b 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -23,6 +23,8 @@ set -ue . $(dirname "$0")/wsrep_sst_common +wait_previous_sst + EINVAL=22 if test -z "$WSREP_SST_OPT_HOST"; then wsrep_log_error "HOST cannot be nil"; exit $EINVAL; fi diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 0ebe8063823..6bf5ba83fd2 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -22,13 +22,22 @@ set -ue # This is a reference script for rsync-based state snapshot transfer . $(dirname "$0")/wsrep_sst_common -wsrep_check_datadir wsrep_check_programs rsync RSYNC_REAL_PID=0 # rsync process id STUNNEL_REAL_PID=0 # stunnel process id +MODULE="${WSREP_SST_OPT_MODULE:-rsync_sst}" + +RSYNC_PID="$DATA/$MODULE.pid" +RSYNC_CONF="$DATA/$MODULE.conf" + +STUNNEL_CONF="$DATA/stunnel.conf" +STUNNEL_PID="$DATA/stunnel.pid" + +MAGIC_FILE="$DATA/rsync_sst_complete" + cleanup_joiner() { # Since this is invoked just after exit NNN @@ -150,13 +159,6 @@ check_pid_and_port() check_pid "$pid_file" && [ $CHECK_PID -eq $pid ] } -DATA="$WSREP_SST_OPT_DATA" - -STUNNEL_CONF="$DATA/stunnel.conf" -STUNNEL_PID="$DATA/stunnel.pid" - -MAGIC_FILE="$DATA/rsync_sst_complete" - get_binlog if [ -n "$WSREP_SST_OPT_BINLOG" ]; then @@ -164,8 +166,6 @@ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi -create_data - BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" ar_log_dir="$DATA_DIR" @@ -270,22 +270,7 @@ fi readonly SECRET_TAG='secret' readonly BYPASS_TAG='bypass' -SST_PID="$DATA/wsrep_sst.pid" - -# give some time for previous SST to complete: -check_round=0 -while check_pid "$SST_PID"; do - wsrep_log_info "Previous SST is not completed, waiting for it to exit" - check_round=$(( check_round+1 )) - if [ $check_round -eq 20 ]; then - wsrep_log_error "previous SST script still running." - exit 114 # EALREADY - fi - sleep 1 -done - -trap simple_cleanup EXIT -echo $$ > "$SST_PID" +wait_previous_sst # give some time for stunnel from the previous SST to complete: check_round=0 @@ -293,26 +278,21 @@ while check_pid "$STUNNEL_PID" 1 "$STUNNEL_CONF"; do wsrep_log_info "Lingering stunnel daemon found at startup," \ "waiting for it to exit" check_round=$(( check_round+1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "stunnel daemon still running." + if [ $check_round -eq 30 ]; then + wsrep_log_error "stunnel daemon still running..." exit 114 # EALREADY fi sleep 1 done -MODULE="${WSREP_SST_OPT_MODULE:-rsync_sst}" - -RSYNC_PID="$DATA/$MODULE.pid" -RSYNC_CONF="$DATA/$MODULE.conf" - # give some time for rsync from the previous SST to complete: check_round=0 while check_pid "$RSYNC_PID" 1 "$RSYNC_CONF"; do wsrep_log_info "Lingering rsync daemon found at startup," \ "waiting for it to exit" check_round=$(( check_round+1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "rsync daemon still running." + if [ $check_round -eq 30 ]; then + wsrep_log_error "rsync daemon still running..." exit 114 # EALREADY fi sleep 1 From 228cb073ad5e85e7444152b581413143aee58568 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Sun, 15 Sep 2024 04:27:23 +0200 Subject: [PATCH 14/17] galera SST scripts: comments update --- scripts/wsrep_sst_backup.sh | 4 ++-- scripts/wsrep_sst_common.sh | 5 +++-- scripts/wsrep_sst_mariabackup.sh | 4 +++- scripts/wsrep_sst_mysqldump.sh | 4 ++-- scripts/wsrep_sst_rsync.sh | 4 ++-- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/wsrep_sst_backup.sh b/scripts/wsrep_sst_backup.sh index e1a8b541cc8..6f72064abd5 100644 --- a/scripts/wsrep_sst_backup.sh +++ b/scripts/wsrep_sst_backup.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2024 MariaDB # Copyright (C) 2010-2014 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -19,7 +19,7 @@ set -ue # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston # MA 02110-1335 USA. -# This is a reference script for rsync-based state snapshot transfer +# This is a reference script for backup recovery state snapshot transfer. . $(dirname "$0")/wsrep_sst_common diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index e6b48c4b6a6..9ea539a4e99 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2022 MariaDB +# Copyright (C) 2017-2024 MariaDB # Copyright (C) 2012-2015 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -15,7 +15,8 @@ # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston # MA 02110-1335 USA. -# This is a common command line parser to be sourced by other SST scripts +# This is a common command line parser and common functions to +# be sourced by other SST scripts. trap 'exit 32' HUP PIPE trap 'exit 3' INT QUIT TERM diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index c2d4752f3f2..4db299f7670 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2022 MariaDB +# Copyright (C) 2017-2024 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -19,6 +19,8 @@ set -ue # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston # MA 02110-1335 USA. +# This is a reference script for mariadb-backup-based state snapshot transfer. + # Documentation: # https://mariadb.com/kb/en/mariabackup-overview/ # Make sure to read that before proceeding! diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 88b7d13cf5b..fa896bf5bb1 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -3,7 +3,7 @@ set -ue # Copyright (C) 2009-2015 Codership Oy -# Copyright (C) 2017-2022 MariaDB +# Copyright (C) 2017-2024 MariaDB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ set -ue # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston # MA 02110-1335 USA. -# This is a reference script for mysqldump-based state snapshot tansfer +# This is a reference script for mysqldump-based state snapshot tansfer. . $(dirname "$0")/wsrep_sst_common diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 6bf5ba83fd2..8acca6f1171 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2022 MariaDB +# Copyright (C) 2017-2024 MariaDB # Copyright (C) 2010-2022 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -19,7 +19,7 @@ set -ue # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston # MA 02110-1335 USA. -# This is a reference script for rsync-based state snapshot transfer +# This is a reference script for rsync-based state snapshot transfer. . $(dirname "$0")/wsrep_sst_common From 64356509afbbe0f1292fb5fd46849a6e735a83f1 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Sun, 15 Sep 2024 04:34:33 +0200 Subject: [PATCH 15/17] galera SST scripts: moving mysqldump-specific code out of the wsrep_sst_common --- scripts/wsrep_sst_common.sh | 13 ------------- scripts/wsrep_sst_mysqldump.sh | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 9ea539a4e99..9a89689ede6 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -865,19 +865,6 @@ readonly WSREP_SST_OPT_ADDR_PORT script_binary=$(dirname "$0") SCRIPTS_DIR=$(cd "$script_binary"; pwd) EXTRA_DIR="$SCRIPTS_DIR/../extra" -CLIENT_DIR="$SCRIPTS_DIR/../client" - -if [ -x "$CLIENT_DIR/mysql" ]; then - MYSQL_CLIENT="$CLIENT_DIR/mysql" -else - MYSQL_CLIENT=$(commandex 'mysql') -fi - -if [ -x "$CLIENT_DIR/mysqldump" ]; then - MYSQLDUMP="$CLIENT_DIR/mysqldump" -else - MYSQLDUMP=$(commandex 'mysqldump') -fi if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then MY_PRINT_DEFAULTS="$SCRIPTS_DIR/my_print_defaults" diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index fa896bf5bb1..f8fbda24356 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -23,6 +23,20 @@ set -ue . $(dirname "$0")/wsrep_sst_common +CLIENT_DIR="$SCRIPTS_DIR/../client" + +if [ -x "$CLIENT_DIR/mysql" ]; then + MYSQL_CLIENT="$CLIENT_DIR/mysql" +else + MYSQL_CLIENT=$(commandex 'mysql') +fi + +if [ -x "$CLIENT_DIR/mysqldump" ]; then + MYSQLDUMP="$CLIENT_DIR/mysqldump" +else + MYSQLDUMP=$(commandex 'mysqldump') +fi + wait_previous_sst EINVAL=22 @@ -37,7 +51,8 @@ if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED" && \ [ "$WSREP_SST_OPT_PORT" = "$WSREP_SST_OPT_LPORT" ] then wsrep_log_error \ - "destination address '$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT' matches source address." + "destination address '$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT'" \ + "matches source address." exit $EINVAL fi From 45be538cf4704e8e9306b4abe9def43d3d375dc6 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Sun, 15 Sep 2024 06:46:53 +0200 Subject: [PATCH 16/17] galera SST scripts: added missing 'datadir' parameter for mysqldump method --- sql/wsrep_sst.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 13500a83f43..34eda4e286d 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -1522,11 +1522,13 @@ static int sst_donate_mysqldump (const char* addr, WSREP_SST_OPT_PORT " '%u' " WSREP_SST_OPT_LPORT " '%u' " WSREP_SST_OPT_SOCKET " '%s' " + WSREP_SST_OPT_DATA " '%s' " "%s" WSREP_SST_OPT_GTID " '%s:%lld,%d-%d-%llu' " WSREP_SST_OPT_GTID_DOMAIN_ID " '%d'" "%s", addr, port, mysqld_port, mysqld_unix_port, + mysql_real_data_home, wsrep_defaults_file, uuid_oss.str().c_str(), gtid.seqno().get(), wsrep_gtid_server.domain_id, wsrep_gtid_server.server_id, From 222744c54e0b9ec6cf4b74cbbd9a81ae6436ee85 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 16 Sep 2024 04:57:56 +0200 Subject: [PATCH 17/17] galera SST scripts: fixing glitchy sockstat issues for FreeBSD --- scripts/wsrep_sst_common.sh | 20 ++++++++++++++------ scripts/wsrep_sst_rsync.sh | 19 +++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 9a89689ede6..f9e563c2a95 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1233,6 +1233,7 @@ check_sockets_utils() # sockstat in FreeBSD is different from other systems, # let's denote it with a different value: sockstat_available=2 + sockstat_opts='-46lq -P tcp -p' fi else socket_utility="$(commandex lsof)" @@ -1277,18 +1278,25 @@ check_port() [ $pid -le 0 ] && pid='[0-9]+' - local rc=1 + local rc=2 # ENOENT if [ $ss_available -ne 0 ]; then $socket_utility $ss_opts -t "( sport = :$port )" 2>/dev/null | \ grep -q -E "[[:space:]]users:[[:space:]]?\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 elif [ $sockstat_available -ne 0 ]; then if [ $sockstat_available -gt 1 ]; then - # sockstat on FreeBSD does not return the connection - # state without special option that cancel filtering - # by the port, so we ignore the connection state for - # this system: - $socket_utility $sockstat_opts "$port" 2>/dev/null | \ + # The sockstat command on FreeBSD does not return + # the connection state without special option, but + # it supports filtering by connection state: + local out + out=$($socket_utility $sockstat_opts "$port" 2>/dev/null) || rc=16 # EBUSY + # On FreeBSD, the sockstat utility may exit without + # any output due to locking issues in certain versions; + # let's return a special exit code in such cases: + if [ $rc -eq 16 -o -z "$out" ]; then + return 16 # EBUSY + fi + echo "$out" | \ grep -q -E "^[^[:space:]]+[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid([[:space:]]|\$)" && rc=0 else $socket_utility $sockstat_opts "$port" 2>/dev/null | \ diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 8acca6f1171..eb68d134eff 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -112,11 +112,11 @@ check_pid_and_port() grep -o -E "([^[:space:]]+[[:space:]]+){4}[^[:space:]]+" || :) else if [ $sockstat_available -gt 1 ]; then - # sockstat on FreeBSD does not return the connection - # state without special option that cancel filtering - # by the port, so we ignore the connection state for - # this system, also on the FreeBSD sockstat utility - # produces an additional column: + # The sockstat command on FreeBSD does not return + # the connection state without special option, but + # it supports filtering by connection state. + # Additionally, the sockstat utility on FreeBSD + # produces an one extra column: port_info=$($socket_utility $sockstat_opts "$port" 2>/dev/null | \ grep -o -E "([^[:space:]]+[[:space:]]+){5}[^[:space:]]+" || :) else @@ -149,7 +149,14 @@ check_pid_and_port() return 1 fi - if ! check_port $pid "$port" "$utils"; then + local rc=0 + check_port $pid "$port" "$utils" || rc=$? + if [ $rc -eq 16 ]; then + # We will ignore the return code EBUSY, which indicates + # a failed attempt to run the utility for retrieving + # socket information (on some systems): + return 1 + elif [ $rc -ne 0 ]; then wsrep_log_error "rsync or stunnel daemon port '$port'" \ "has been taken by another program" exit 16 # EBUSY