MDEV-9312: storage engine not enforced during galera cluster replication

Perform a post initialization of plugin-related variables
of wsrep threads after their global counterparts have been
initialized.

mysql-test/suite/galera/r/enforce_storage_engine2.result

@@ -0,0 +1,24 @@
+#
+# MDEV-9312: storage engine not enforced during galera cluster
+# replication
+#
+CREATE TABLE t1(i INT) ENGINE=INNODB;
+CREATE TABLE t2(i INT) ENGINE=MYISAM;
+Warnings:
+Note	1266	Using storage engine InnoDB for table 't2'
+SHOW TABLES;
+Tables_in_test
+t1
+t2
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `i` int(11) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SHOW CREATE TABLE t2;
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `i` int(11) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+DROP TABLE t1, t2;
+# End of tests

mysql-test/suite/galera/t/enforce_storage_engine2.opt

@@ -0,0 +1,2 @@
+--enforce_storage_engine=innodb --sql_mode=''
+

mysql-test/suite/galera/t/enforce_storage_engine2.test

@@ -0,0 +1,20 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+--echo #
+--echo # MDEV-9312: storage engine not enforced during galera cluster
+--echo # replication
+--echo #
+--connection node_1
+CREATE TABLE t1(i INT) ENGINE=INNODB;
+CREATE TABLE t2(i INT) ENGINE=MYISAM;
+
+--connection node_2
+SHOW TABLES;
+SHOW CREATE TABLE t1;
+SHOW CREATE TABLE t2;
+
+# Cleanup
+DROP TABLE t1, t2;
+
+--echo # End of tests

sql/mysqld.cc

@@ -5171,6 +5171,12 @@ static int init_server_components()
       opt_bin_logname= my_once_strdup(buf, MYF(MY_WME));
   }
 
+  /*
+    Since some wsrep threads (THDs) are create before plugins are
+    initialized, LOCK_plugin mutex needs to be initialized here.
+  */
+  plugin_mutex_init();
+
   /*
     Wsrep initialization must happen at this point, because:
     - opt_bin_logname must be known when starting replication
@@ -5412,6 +5418,15 @@ static int init_server_components()
 #endif
 
 #ifdef WITH_WSREP
+  /*
+    Now is the right time to initialize members of wsrep startup threads
+    that rely on plugins and other related global system variables to be
+    initialized. This initialization was not possible before, as plugins
+    (and thus some global system variables) are initialized after wsrep
+    startup threads are created.
+  */
+  wsrep_plugins_post_init();
+
   if (WSREP_ON && !opt_bin_log)
   {
     wsrep_emulate_bin_log= 1;

sql/sql_plugin.cc

@@ -1555,9 +1555,6 @@ int plugin_init(int *argc, char **argv, int flags)
                    get_bookmark_hash_key, NULL, HASH_UNIQUE))
       goto err;
 
-
-  mysql_mutex_init(key_LOCK_plugin, &LOCK_plugin, MY_MUTEX_INIT_FAST);
-
   /*
     The 80 is from 2016-04-27 when we had 71 default plugins
     Big enough to avoid many mallocs even in future
@@ -3138,28 +3135,19 @@ void plugin_thdvar_init(THD *thd)
   thd->variables.dynamic_variables_size= 0;
   thd->variables.dynamic_variables_ptr= 0;
 
-  if (IF_WSREP((!WSREP(thd) || !thd->wsrep_applier),1))
+  mysql_mutex_lock(&LOCK_plugin);
-  {
+  thd->variables.table_plugin=
-    mysql_mutex_lock(&LOCK_plugin);
+      intern_plugin_lock(NULL, global_system_variables.table_plugin);
-    thd->variables.table_plugin=
+  if (global_system_variables.tmp_table_plugin)
-        intern_plugin_lock(NULL, global_system_variables.table_plugin);
+    thd->variables.tmp_table_plugin=
-    if (global_system_variables.tmp_table_plugin)
+          intern_plugin_lock(NULL, global_system_variables.tmp_table_plugin);
-      thd->variables.tmp_table_plugin=
+  if (global_system_variables.enforced_table_plugin)
-            intern_plugin_lock(NULL, global_system_variables.tmp_table_plugin);
+    thd->variables.enforced_table_plugin=
-    if (global_system_variables.enforced_table_plugin)
+          intern_plugin_lock(NULL, global_system_variables.enforced_table_plugin);
-      thd->variables.enforced_table_plugin=
+  intern_plugin_unlock(NULL, old_table_plugin);
-            intern_plugin_lock(NULL, global_system_variables.enforced_table_plugin);
+  intern_plugin_unlock(NULL, old_tmp_table_plugin);
-    intern_plugin_unlock(NULL, old_table_plugin);
+  intern_plugin_unlock(NULL, old_enforced_table_plugin);
-    intern_plugin_unlock(NULL, old_tmp_table_plugin);
+  mysql_mutex_unlock(&LOCK_plugin);
-    intern_plugin_unlock(NULL, old_enforced_table_plugin);
-    mysql_mutex_unlock(&LOCK_plugin);
-  }
-  else
-  {
-    thd->variables.table_plugin= NULL;
-    thd->variables.tmp_table_plugin= NULL;
-    thd->variables.enforced_table_plugin= NULL;
-  }
 
   DBUG_VOID_RETURN;
 }
@@ -4298,3 +4286,51 @@ int thd_setspecific(MYSQL_THD thd, MYSQL_THD_KEY_T key, void *value)
   return 0;
 }
 
+void plugin_mutex_init()
+{
+  mysql_mutex_init(key_LOCK_plugin, &LOCK_plugin, MY_MUTEX_INIT_FAST);
+}
+
+#ifdef WITH_WSREP
+
+/*
+  Placeholder for global_system_variables.table_plugin required during
+  initialization of startup wsrep threads.
+*/
+static st_plugin_int wsrep_dummy_plugin;
+static st_plugin_int *wsrep_dummy_plugin_ptr;
+
+/*
+  Initialize wsrep_dummy_plugin and assign it to
+  global_system_variables.table_plugin.
+*/
+void wsrep_plugins_pre_init()
+{
+  wsrep_dummy_plugin_ptr= &wsrep_dummy_plugin;
+  wsrep_dummy_plugin.state= PLUGIN_IS_DISABLED;
+  global_system_variables.table_plugin=
+    plugin_int_to_ref(wsrep_dummy_plugin_ptr);
+}
+
+/*
+  This function is intended to be called after the plugins and related
+  global system variables are initialized. It re-initializes some data
+  members of wsrep startup threads with correct values, as these value
+  were not available at the time these threads were created.
+*/
+void wsrep_plugins_post_init()
+{
+  THD *thd;
+  I_List_iterator<THD> it(threads);
+
+  while ((thd= it++))
+  {
+    if (IF_WSREP(thd->wsrep_applier,1))
+    {
+      plugin_thdvar_init(thd);
+    }
+  }
+
+  return;
+}
+#endif /* WITH_WSREP */

sql/sql_plugin.h

@@ -182,6 +182,7 @@ sys_var *find_plugin_sysvar(st_plugin_int *plugin, st_mysql_sys_var *var);
 void plugin_opt_set_limits(struct my_option *, const struct st_mysql_sys_var *);
 extern SHOW_COMP_OPTION plugin_status(const char *name, size_t len, int type);
 extern bool check_valid_path(const char *path, size_t length);
+extern void plugin_mutex_init();
 
 typedef my_bool (plugin_foreach_func)(THD *thd,
                                       plugin_ref plugin,
@@ -199,3 +200,9 @@ sys_var *find_sys_var_ex(THD *thd, const char *str, size_t length,
 
 extern void sync_dynamic_session_variables(THD* thd, bool global_lock);
 #endif
+
+#ifdef WITH_WSREP
+extern void wsrep_plugins_pre_init();
+extern void wsrep_plugins_post_init();
+#endif /* WITH_WSREP */
+

sql/wsrep_mysqld.cc

@@ -36,6 +36,7 @@
 #include <cstdlib>
 #include "log_event.h"
 #include <slave.h>
+#include "sql_plugin.h"                         /* wsrep_plugins_pre_init() */
 
 wsrep_t *wsrep                  = NULL;
 /*
@@ -792,7 +793,6 @@ void wsrep_thr_init()
   DBUG_VOID_RETURN;
 }
 
-
 void wsrep_init_startup (bool first)
 {
   if (wsrep_init()) unireg_abort(1);
@@ -803,6 +803,13 @@ void wsrep_init_startup (bool first)
      wsrep_debug, wsrep_convert_LOCK_to_trx,
      (wsrep_on_fun)wsrep_on);
 
+  /*
+    Pre-initialize global_system_variables.table_plugin with a dummy engine
+    (placeholder) required during the initialization of wsrep threads (THDs).
+    (see: plugin_thdvar_init())
+  */
+  wsrep_plugins_pre_init();
+
   /* Skip replication start if dummy wsrep provider is loaded */
   if (!strcmp(wsrep_provider, WSREP_NONE)) return;