From 94ef07d61edd21d81eaa788c07a15e5508aacbe0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@galeracluster.com>
Date: Tue, 24 Sep 2024 11:09:00 +0300
Subject: [PATCH] MDEV-32631 : galera_2_cluster: before_rollback(): Assertion
 `0' failed

Test case changes only. Add wait_conditions to make sure
nodes rejoin the cluster. Assertion itself should not be
possible anymore as we do not allow sequences on
Aria tables.

Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>
---
 .../galera/r/galera_sequences,binlogoff.rdiff | 11 +++
 .../suite/galera/r/galera_sequences.result    | 13 ++++
 .../suite/galera/t/galera_sequences.cnf       |  4 -
 .../galera/t/galera_sequences.combinations    |  5 ++
 .../suite/galera/t/galera_sequences.test      | 24 ++++++
 mysql-test/suite/galera_3nodes/disabled.def   |  2 -
 .../galera_3nodes/r/galera_2_cluster.result   | 32 +++++++-
 .../galera_3nodes/t/galera_2_cluster.cnf      |  9 +++
 .../t/galera_2_cluster.combinations           |  5 ++
 .../galera_3nodes/t/galera_2_cluster.test     | 74 +++++++++++++++----
 10 files changed, 155 insertions(+), 24 deletions(-)
 create mode 100644 mysql-test/suite/galera/r/galera_sequences,binlogoff.rdiff
 create mode 100644 mysql-test/suite/galera/t/galera_sequences.combinations
 create mode 100644 mysql-test/suite/galera_3nodes/t/galera_2_cluster.combinations

diff --git a/mysql-test/suite/galera/r/galera_sequences,binlogoff.rdiff b/mysql-test/suite/galera/r/galera_sequences,binlogoff.rdiff
new file mode 100644
index 00000000000..425d1175a09
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_sequences,binlogoff.rdiff
@@ -0,0 +1,11 @@
+--- r/galera_sequences.result
++++ r/galera_sequences,binlogoff.reject
+@@ -313,7 +313,7 @@
+ 7	4
+ SELECT NEXTVAL(t);
+ NEXTVAL(t)
+-42
++2
+ connection node_1;
+ DROP TABLE t1;
+ DROP SEQUENCE t;
diff --git a/mysql-test/suite/galera/r/galera_sequences.result b/mysql-test/suite/galera/r/galera_sequences.result
index 1f6b2bd6637..e78b89d4b09 100644
--- a/mysql-test/suite/galera/r/galera_sequences.result
+++ b/mysql-test/suite/galera/r/galera_sequences.result
@@ -279,6 +279,9 @@ NEXTVAL(t)
 connection node_1;
 DROP TABLE t1;
 DROP SEQUENCE t;
+connection node_2;
+SET SESSION wsrep_sync_wait=15;
+connection node_1;
 CREATE SEQUENCE t INCREMENT BY 0 CACHE=20 ENGINE=INNODB;
 CREATE TABLE t1(a int not null primary key default nextval(t), b int) engine=innodb;
 BEGIN;
@@ -324,4 +327,14 @@ ERROR 42000: You have an error in your SQL syntax; check the manual that corresp
 ALTER SEQUENCE IF EXISTS t MINVALUE=1;
 ERROR 42000: This version of MariaDB doesn't yet support 'CACHE without INCREMENT BY 0 in Galera cluster'
 DROP TABLE t;
+
+MDEV-32631:
+
+CREATE OR REPLACE TABLE t1(c INT ) ENGINE=ARIA;
+SET SESSION WSREP_OSU_METHOD=RSU;
+INSERT INTO t1 SELECT seq,concat(seq,1) FROM seq_1_to_100;
+ERROR 42000: This version of MariaDB doesn't yet support 'RSU on this table engine'
+SET SESSION WSREP_OSU_METHOD=TOI;
+DROP TABLE t1;
+
 End of 10.5 tests
diff --git a/mysql-test/suite/galera/t/galera_sequences.cnf b/mysql-test/suite/galera/t/galera_sequences.cnf
index 3a0543e3d34..8701e86db5f 100644
--- a/mysql-test/suite/galera/t/galera_sequences.cnf
+++ b/mysql-test/suite/galera/t/galera_sequences.cnf
@@ -1,13 +1,9 @@
 !include ../galera_2nodes.cnf
 
 [mysqld.1]
-log-bin
-log-slave-updates
 auto-increment-increment=2
 auto-increment-offset=1
 
 [mysqld.2]
-log-bin
-log-slave-updates
 auto-increment-increment=2
 auto-increment-offset=2
diff --git a/mysql-test/suite/galera/t/galera_sequences.combinations b/mysql-test/suite/galera/t/galera_sequences.combinations
new file mode 100644
index 00000000000..cef98e75213
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_sequences.combinations
@@ -0,0 +1,5 @@
+[binlogon]
+log-bin
+log-slave-updates
+
+[binlogoff]
diff --git a/mysql-test/suite/galera/t/galera_sequences.test b/mysql-test/suite/galera/t/galera_sequences.test
index f99796264b6..db388de2551 100644
--- a/mysql-test/suite/galera/t/galera_sequences.test
+++ b/mysql-test/suite/galera/t/galera_sequences.test
@@ -1,6 +1,7 @@
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 --source include/have_sequence.inc
+--source include/have_aria.inc
 
 #
 # MDEV-19353 : Alter Sequence do not replicate to another nodes with in Galera Cluster
@@ -317,6 +318,12 @@ SELECT NEXTVAL(t);
 DROP TABLE t1;
 DROP SEQUENCE t;
 
+--connection node_2
+--let $wsrep_sync_wait_orig_2 = `SELECT @@wsrep_sync_wait`
+SET SESSION wsrep_sync_wait=15;
+
+--connection node_1
+
 CREATE SEQUENCE t INCREMENT BY 0 CACHE=20 ENGINE=INNODB;
 CREATE TABLE t1(a int not null primary key default nextval(t), b int) engine=innodb;
 #
@@ -339,6 +346,10 @@ SELECT NEXTVAL(t);
 SELECT * FROM t1;
 SELECT NEXTVAL(t);
 
+--disable_query_log
+--eval SET SESSION wsrep_sync_wait = $wsrep_sync_wait_orig_2
+--enable_query_log
+
 --connection node_1
 DROP TABLE t1;
 DROP SEQUENCE t;
@@ -356,4 +367,17 @@ CREATE TABLE t1 (c VARCHAR) ENGINE=InnoDB;
 ALTER SEQUENCE IF EXISTS t MINVALUE=1;
 
 DROP TABLE t;
+
+--echo
+--echo MDEV-32631:
+--echo
+
+CREATE OR REPLACE TABLE t1(c INT ) ENGINE=ARIA;
+SET SESSION WSREP_OSU_METHOD=RSU;
+--error ER_NOT_SUPPORTED_YET
+INSERT INTO t1 SELECT seq,concat(seq,1) FROM seq_1_to_100;
+SET SESSION WSREP_OSU_METHOD=TOI;
+DROP TABLE t1;
+
+--echo
 --echo End of 10.5 tests
diff --git a/mysql-test/suite/galera_3nodes/disabled.def b/mysql-test/suite/galera_3nodes/disabled.def
index 6fbfda9a07f..fcaf38a3d7b 100644
--- a/mysql-test/suite/galera_3nodes/disabled.def
+++ b/mysql-test/suite/galera_3nodes/disabled.def
@@ -9,5 +9,3 @@
 #  Do not use any TAB characters for whitespace.
 #
 ##############################################################################
-
-galera_2_cluster : MDEV-32631 galera_2_cluster: before_rollback(): Assertion `0' failed
diff --git a/mysql-test/suite/galera_3nodes/r/galera_2_cluster.result b/mysql-test/suite/galera_3nodes/r/galera_2_cluster.result
index 0a2be393d27..9a151c10b9b 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_2_cluster.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_2_cluster.result
@@ -1,5 +1,6 @@
 connection node_2;
 connection node_1;
+connect node_6, 127.0.0.1, root, , test, $NODE_MYPORT_6;
 connect node_5, 127.0.0.1, root, , test, $NODE_MYPORT_5;
 connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4;
 connection node_4;
@@ -20,7 +21,6 @@ include/sync_with_master_gtid.inc
 SELECT COUNT(*) = 1 FROM t1;
 COUNT(*) = 1
 1
-connect node_6, 127.0.0.1, root, , test, $NODE_MYPORT_6;
 connection node_6;
 SELECT COUNT(*) = 1 FROM t1;
 COUNT(*) = 1
@@ -50,12 +50,22 @@ SELECT COUNT(*) = 3 FROM information_schema.columns WHERE table_name ='t1';
 COUNT(*) = 3
 1
 connection node_2;
+connection node_1;
+connection node_3;
+connection node_4;
+connection node_5;
+connection node_6;
+connection node_2;
 OPTIMIZE TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	note	Table does not support optimize, doing recreate + analyze instead
 test.t1	optimize	status	OK
+Warnings:
+Note	1592	Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system variable that may have a different value on the slave
 connection node_1;
+connection node_3;
 connection node_4;
+connection node_5;
 connection node_6;
 connection node_1;
 DROP TABLE t1;
@@ -74,19 +84,33 @@ connection node_2;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
-CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
 connection node_3;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
-CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
 connection node_5;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
-CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
 connection node_6;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+connection node_1;
 CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+connection node_3;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+connection node_4;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+connection node_5;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+connection node_6;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
diff --git a/mysql-test/suite/galera_3nodes/t/galera_2_cluster.cnf b/mysql-test/suite/galera_3nodes/t/galera_2_cluster.cnf
index 41c1c7c0519..297e0fad159 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_2_cluster.cnf
+++ b/mysql-test/suite/galera_3nodes/t/galera_2_cluster.cnf
@@ -1,25 +1,34 @@
 !include ../galera_2x3nodes.cnf
 
+[mysqld]
+wsrep-debug=1
+
 [mysqld.1]
 wsrep_gtid_domain_id=1
 server-id=11
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT20S;evs.inactive_timeout=PT30S;evs.install_timeout=PT25S;pc.wait_prim_timeout=PT60S;gcache.size=128M;pc.weight=2'
 
 [mysqld.2]
 wsrep_gtid_domain_id=1
 server-id=12
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT20S;evs.inactive_timeout=PT30S;evs.install_timeout=PT25S;pc.wait_prim_timeout=PT60S;gcache.size=128M'
 
 [mysqld.3]
 wsrep_gtid_domain_id=1
 server-id=13
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#galera_port;evs.suspect_timeout=PT20S;evs.inactive_timeout=PT30S;evs.install_timeout=PT25S;pc.wait_prim_timeout=PT60S;gcache.size=128M'
 
 [mysqld.4]
 wsrep_gtid_domain_id=2
 server-id=21
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.4.#galera_port;evs.suspect_timeout=PT20S;evs.inactive_timeout=PT30S;evs.install_timeout=PT25S;pc.wait_prim_timeout=PT60S;gcache.size=128M'
 
 [mysqld.5]
 wsrep_gtid_domain_id=2
 server-id=22
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.5.#galera_port;evs.suspect_timeout=PT20S;evs.inactive_timeout=PT30S;evs.install_timeout=PT25S;pc.wait_prim_timeout=PT60S;gcache.size=128M'
 
 [mysqld.6]
 wsrep_gtid_domain_id=2
 server-id=23
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.6.#galera_port;evs.suspect_timeout=PT20S;evs.inactive_timeout=PT30S;evs.install_timeout=PT25S;pc.wait_prim_timeout=PT60S;gcache.size=128M'
diff --git a/mysql-test/suite/galera_3nodes/t/galera_2_cluster.combinations b/mysql-test/suite/galera_3nodes/t/galera_2_cluster.combinations
new file mode 100644
index 00000000000..cef98e75213
--- /dev/null
+++ b/mysql-test/suite/galera_3nodes/t/galera_2_cluster.combinations
@@ -0,0 +1,5 @@
+[binlogon]
+log-bin
+log-slave-updates
+
+[binlogoff]
diff --git a/mysql-test/suite/galera_3nodes/t/galera_2_cluster.test b/mysql-test/suite/galera_3nodes/t/galera_2_cluster.test
index 86119bf7bcb..a41a5096c7d 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_2_cluster.test
+++ b/mysql-test/suite/galera_3nodes/t/galera_2_cluster.test
@@ -9,7 +9,9 @@
 --source include/big_test.inc
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
+--source include/force_restart.inc
 
+--connect node_6, 127.0.0.1, root, , test, $NODE_MYPORT_6
 --connect node_5, 127.0.0.1, root, , test, $NODE_MYPORT_5
 --connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4
 
@@ -43,7 +45,6 @@ SELECT COUNT(*) = 1 FROM t1;
 
 SELECT COUNT(*) = 1 FROM t1;
 
---connect node_6, 127.0.0.1, root, , test, $NODE_MYPORT_6
 --connection node_6
 
 SELECT COUNT(*) = 1 FROM t1;
@@ -82,23 +83,46 @@ SELECT COUNT(*) = 3 FROM information_schema.columns WHERE table_name ='t1';
 #
 
 --connection node_2
-
---let $wsrep_last_committed_before = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'`
-OPTIMIZE TABLE t1;
+--let $wsrep_last_committed_before_2 = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'`
 
 --connection node_1
+--let $wsrep_last_committed_before_1 = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'`
 
---let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
+--connection node_3
+--let $wsrep_last_committed_before_3 = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'`
+
+--connection node_4
+--let $wsrep_last_committed_before_4 = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'`
+
+--connection node_5
+--let $wsrep_last_committed_before_5 = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'`
+
+--connection node_6
+--let $wsrep_last_committed_before_6 = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'`
+
+--connection node_2
+OPTIMIZE TABLE t1;
+--let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before_2 + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
+--source include/wait_condition.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before_1 + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
+--source include/wait_condition.inc
+
+--connection node_3
+--let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before_3 + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
 --source include/wait_condition.inc
 
 --connection node_4
+--let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before_4 + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
+--source include/wait_condition.inc
 
---let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
+--connection node_5
+--let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before_5 + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
 --source include/wait_condition.inc
 
 --connection node_6
-
---let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
+--let $wait_condition = SELECT VARIABLE_VALUE >= $wsrep_last_committed_before_6 + 1 FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
 --source include/wait_condition.inc
 
 #
@@ -116,6 +140,7 @@ RESET SLAVE;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+--source include/wait_until_ready.inc
 SET GLOBAL GTID_SLAVE_POS="";
 
 --connection node_1
@@ -123,35 +148,56 @@ SET GLOBAL GTID_SLAVE_POS="";
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+--source include/wait_until_ready.inc
 
 --connection node_2
 
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
-
-CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+--source include/wait_until_ready.inc
 
 --connection node_3
 
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
-
-CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+--source include/wait_until_ready.inc
 
 --connection node_5
 
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
-
-CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+--source include/wait_until_ready.inc
 
 --connection node_6
 
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+--source include/wait_until_ready.inc
 
+connection node_1;
 CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+
+connection node_3;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+
+connection node_4;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+
+connection node_5;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");
+
+connection node_6;
+CALL mtr.add_suppression("Ignoring server id .* for non bootstrap node");
+CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since ");