MDEV-34822: Skip FK checks in Galera during applying in IST

Appliers need to verify foreign key constraints during normal
operation, in multi-active topologies, and for this reason appliers
are configured to enable FK checking.

However, during node joining, in IST and latter catch up period,
the node is still idle (from local connections), and only source
for incoming transactions is the cluster sending certified write
sets for applying. IST happens with parallel applying, and there
is a possibility that foreign key check cause lock conflicts between
appliers accessing FK child and parent tables. Also, the excessive
FK checking will slow down IST process somewhat.

For this reasons, we could relax FK checks for appliers during IST
and catch up periods. The relaxed FK check mode should, however, be
configurable e.g. by wsrep_mode flag: SKIP_APPLIER_FK_CHECKS_IN_IST.
When this operation mode is set, and the node is processing IST or
catch up, appliers should skip FK checking.

Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

mysql-test/suite/galera/r/galera_skip_fk_check_ist.result

@@ -0,0 +1,102 @@
+connection node_4;
+connection node_3;
+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+connection node_3;
+connection node_4;
+connection node_1;
+CREATE TABLE parent (
+id INT PRIMARY KEY
+) ENGINE=InnoDB;
+CREATE TABLE child (
+id INT PRIMARY KEY,
+parent_id INT,
+KEY (parent_id),
+CONSTRAINT FOREIGN KEY (parent_id) REFERENCES parent(id)
+) ENGINE=InnoDB;
+INSERT INTO parent VALUES (1), (2);
+connection node_4;
+SET SESSION wsrep_on = OFF;
+DELETE FROM parent;
+SET SESSION wsrep_on = ON;
+Shutting down server 4
+connection node_1;
+SET SESSION wsrep_on = ON;
+SET SESSION wsrep_sync_wait = 15;
+connection node_2;
+SET SESSION wsrep_on = ON;
+SET SESSION wsrep_sync_wait = 15;
+connection node_3;
+SET SESSION wsrep_on = ON;
+SET SESSION wsrep_sync_wait = 15;
+Server 4 left the cluster
+connection node_1;
+INSERT INTO child VALUES (1, 1);
+connection node_4;
+Restarting server 4 with disabled FK checks during IST
+include/assert_grep.inc [no FK constraint failure]
+Server 4
+SELECT COUNT(*) AS EXPECT_0 FROM parent;
+EXPECT_0
+0
+SELECT COUNT(*) AS EXPECT_1 FROM child;
+EXPECT_1
+1
+connection node_1;
+Server 1
+SELECT COUNT(*) AS EXPECT_2 FROM parent;
+EXPECT_2
+2
+SELECT COUNT(*) AS EXPECT_1 FROM child;
+EXPECT_1
+1
+connection node_2;
+Server 2
+SELECT COUNT(*) AS EXPECT_2 FROM parent;
+EXPECT_2
+2
+SELECT COUNT(*) AS EXPECT_1 FROM child;
+EXPECT_1
+1
+connection node_3;
+Server 3
+SELECT COUNT(*) AS EXPECT_2 FROM parent;
+EXPECT_2
+2
+SELECT COUNT(*) AS EXPECT_1 FROM child;
+EXPECT_1
+1
+Causing server 4 inconsistency with failed FK check on apply
+INSERT INTO child VALUES (2, 2);
+connection node_1;
+SET SESSION wsrep_on = ON;
+SET SESSION wsrep_sync_wait = 15;
+connection node_2;
+SET SESSION wsrep_on = ON;
+SET SESSION wsrep_sync_wait = 15;
+connection node_3;
+SET SESSION wsrep_on = ON;
+SET SESSION wsrep_sync_wait = 15;
+Server 4 is non-primary
+connection node_4;
+SET SESSION wsrep_on = OFF;
+include/assert_grep.inc [FK constraint failure]
+Restarting server 4 with enabled FK checks during IST
+SELECT COUNT(*) AS EXPECT_2 FROM parent;
+EXPECT_2
+2
+SELECT COUNT(*) AS EXPECT_2 FROM child;
+EXPECT_2
+2
+CALL mtr.add_suppression("Slave SQL: Could not execute Write_rows_v1 event");
+CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed");
+CALL mtr.add_suppression("Inconsistency detected: Inconsistent by consensus");
+CALL mtr.add_suppression("Failed to apply write set");
+DROP TABLE child;
+DROP TABLE parent;
+disconnect node_4;
+disconnect node_3;
+disconnect node_2;
+disconnect node_1;

mysql-test/suite/galera/t/galera_skip_fk_check_ist.cnf

@@ -0,0 +1,16 @@
+!include ../galera_4nodes.cnf
+
+[mysqld.1]
+wsrep_node_name='node_1'
+
+[mysqld.2]
+wsrep_node_name='node_2'
+
+[mysqld.3]
+wsrep_node_name='node_3'
+
+[mysqld.4]
+wsrep_node_name='node_4'
+
+[ENV]
+galera_cluster_size=4

mysql-test/suite/galera/t/galera_skip_fk_check_ist.test

@@ -0,0 +1,136 @@
+#
+# MDEV-34822: Skip FK checks in Galera during applying in IST.
+#
+
+--source include/have_innodb.inc
+--source include/galera_cluster.inc
+--source include/big_test.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--let $node_4=node_4
+--source include/auto_increment_offset_save.inc
+
+# Create parent and child tables.
+--connection node_1
+CREATE TABLE parent (
+	id INT PRIMARY KEY
+) ENGINE=InnoDB;
+
+CREATE TABLE child (
+	id INT PRIMARY KEY,
+	parent_id INT,
+	KEY (parent_id),
+	CONSTRAINT FOREIGN KEY (parent_id) REFERENCES parent(id)
+) ENGINE=InnoDB;
+
+# Fill the parent table with rows that will later be used by the child.
+INSERT INTO parent VALUES (1), (2);
+
+# Wait until the row is replicated on node #4.
+--connection node_4
+--let $wait_condition = SELECT COUNT(*) = 2 FROM parent
+--source include/wait_condition.inc
+
+# Clear the parent table on node #4 and leave the cluster.
+SET SESSION wsrep_on = OFF;
+DELETE FROM parent;
+SET SESSION wsrep_on = ON;
+--echo Shutting down server 4
+--source include/shutdown_mysqld.inc
+
+# Wait for node #4 to leave the cluster.
+--let $members = 3
+--connection node_1
+--source include/wsrep_wait_membership.inc
+--connection node_2
+--source include/wsrep_wait_membership.inc
+--connection node_3
+--source include/wsrep_wait_membership.inc
+--echo Server 4 left the cluster
+
+# Insert a child row that will be sent to node #4 with IST.
+--connection node_1
+INSERT INTO child VALUES (1, 1);
+
+--connection node_4
+--echo Restarting server 4 with disabled FK checks during IST
+--let $start_mysqld_params = --wsrep_mode=APPLIER_SKIP_FK_CHECKS_IN_IST
+--source include/start_mysqld.inc
+
+--let $assert_select = foreign key constraint fails
+--let $assert_count = 0
+--let $assert_text = no FK constraint failure
+--let $assert_only_after = CURRENT_TEST
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.4.err
+--source include/assert_grep.inc
+
+# Child row insert is applied even though there's no parent row.
+--echo Server 4
+SELECT COUNT(*) AS EXPECT_0 FROM parent;
+SELECT COUNT(*) AS EXPECT_1 FROM child;
+
+# Check other nodes have both parent and child rows.
+--connection node_1
+--echo Server 1
+SELECT COUNT(*) AS EXPECT_2 FROM parent;
+SELECT COUNT(*) AS EXPECT_1 FROM child;
+
+--connection node_2
+--echo Server 2
+SELECT COUNT(*) AS EXPECT_2 FROM parent;
+SELECT COUNT(*) AS EXPECT_1 FROM child;
+
+--connection node_3
+--echo Server 3
+SELECT COUNT(*) AS EXPECT_2 FROM parent;
+SELECT COUNT(*) AS EXPECT_1 FROM child;
+
+# Test part below ensures that regular apply still fails on FK check.
+
+--echo Causing server 4 inconsistency with failed FK check on apply
+INSERT INTO child VALUES (2, 2);
+
+# Wait for node #4 to become inconsistent and leave the primary component.
+--let $members = 3
+--connection node_1
+--source include/wsrep_wait_membership.inc
+--connection node_2
+--source include/wsrep_wait_membership.inc
+--connection node_3
+--source include/wsrep_wait_membership.inc
+--echo Server 4 is non-primary
+
+--connection node_4
+SET SESSION wsrep_on = OFF;
+--let $assert_select = foreign key constraint fails
+# Exact count may depend on the log level.
+--let $assert_count =
+--let $assert_match = foreign key constraint fails
+--let $assert_text = FK constraint failure
+--let $assert_only_after = CURRENT_TEST
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.4.err
+--source include/assert_grep.inc
+
+--echo Restarting server 4 with enabled FK checks during IST
+--let $start_mysqld_params =
+--source include/restart_mysqld.inc
+
+# Now everything is in sync.
+SELECT COUNT(*) AS EXPECT_2 FROM parent;
+SELECT COUNT(*) AS EXPECT_2 FROM child;
+
+CALL mtr.add_suppression("Slave SQL: Could not execute Write_rows_v1 event");
+CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed");
+CALL mtr.add_suppression("Inconsistency detected: Inconsistent by consensus");
+CALL mtr.add_suppression("Failed to apply write set");
+
+DROP TABLE child;
+DROP TABLE parent;
+
+# Restore original auto_increment_offset values.
+--source include/auto_increment_offset_restore.inc
+
+--source include/galera_end.inc

mysql-test/suite/sys_vars/r/sysvars_wsrep.result

@@ -327,7 +327,7 @@ VARIABLE_COMMENT	Set of WSREP features that are enabled
 NUMERIC_MIN_VALUE	NULL
 NUMERIC_MAX_VALUE	NULL
 NUMERIC_BLOCK_SIZE	NULL
-ENUM_VALUE_LIST	STRICT_REPLICATION,BINLOG_ROW_FORMAT_ONLY,REQUIRED_PRIMARY_KEY,REPLICATE_MYISAM,REPLICATE_ARIA,DISALLOW_LOCAL_GTID,BF_ABORT_MARIABACKUP
+ENUM_VALUE_LIST	STRICT_REPLICATION,BINLOG_ROW_FORMAT_ONLY,REQUIRED_PRIMARY_KEY,REPLICATE_MYISAM,REPLICATE_ARIA,DISALLOW_LOCAL_GTID,BF_ABORT_MARIABACKUP,APPLIER_SKIP_FK_CHECKS_IN_IST
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 GLOBAL_VALUE_PATH	NULL

sql/sys_vars.cc

@@ -6518,6 +6518,7 @@ static const char *wsrep_mode_names[]=
   "REPLICATE_ARIA",
   "DISALLOW_LOCAL_GTID",
   "BF_ABORT_MARIABACKUP",
+  "APPLIER_SKIP_FK_CHECKS_IN_IST",
   NullS
 };
 static Sys_var_set Sys_wsrep_mode(

sql/wsrep_high_priority_service.cc

@@ -115,7 +115,9 @@ static void wsrep_setup_uk_and_fk_checks(THD* thd)
   else
     thd->variables.option_bits&= ~OPTION_RELAXED_UNIQUE_CHECKS;
 
-  if (wsrep_slave_FK_checks == FALSE)
+  if (wsrep_slave_FK_checks == FALSE ||
+      (wsrep_check_mode(WSREP_MODE_APPLIER_SKIP_FK_CHECKS_IN_IST) &&
+       !wsrep_ready_get()))
     thd->variables.option_bits|= OPTION_NO_FOREIGN_KEY_CHECKS;
   else
     thd->variables.option_bits&= ~OPTION_NO_FOREIGN_KEY_CHECKS;

sql/wsrep_mysqld.h

@@ -131,7 +131,8 @@ enum enum_wsrep_mode {
   WSREP_MODE_REPLICATE_MYISAM= (1ULL << 3),
   WSREP_MODE_REPLICATE_ARIA= (1ULL << 4),
   WSREP_MODE_DISALLOW_LOCAL_GTID= (1ULL << 5),
-  WSREP_MODE_BF_MARIABACKUP= (1ULL << 6)
+  WSREP_MODE_BF_MARIABACKUP= (1ULL << 6),
+  WSREP_MODE_APPLIER_SKIP_FK_CHECKS_IN_IST= (1ULL << 7)
 };
 
 // Streaming Replication