MDEV-29281 Report events from provider (add node eviction event)

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/galera/r/galera_event_node_evict.result

@@ -0,0 +1,21 @@
+connection node_2;
+connection node_1;
+# Correct Galera library found
+connection node_1;
+CALL mtr.add_suppression("\\[Warning\\] WSREP: evicting member .* at .* permanently from group");
+connection node_2;
+CALL mtr.add_suppression("\\[Warning\\] WSREP: handshake with .* .* failed: 'evicted'");
+CALL mtr.add_suppression("\\[ERROR\\] WSREP: exception from gcomm, backend must be restarted: this node has been evicted out of the cluster, gcomm backend restart is required \\(FATAL\\)");
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+CALL mtr.add_suppression("\\[Warning\\] WSREP: evicting member .* at .* permanently from group");
+connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4;
+CALL mtr.add_suppression("\\[Warning\\] WSREP: evicting member .* at .* permanently from group");
+connection node_1;
+connection node_2;
+connection node_3;
+connection node_4;
+connection node_2;
+SET SESSION wsrep_on = ON;
+SET SESSION wsrep_sync_wait = 15;
+SET GLOBAL wsrep_on = OFF;
+include/assert_grep.inc [Node evicted]

mysql-test/suite/galera/t/galera_event_node_evict.cnf

@@ -0,0 +1,14 @@
+!include ../galera_4nodes.cnf
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;evs.auto_evict=1'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;evs.auto_evict=1'
+wsrep_status_file='status2.json'
+
+[mysqld.3]
+wsrep_provider_options='base_port=@mysqld.3.#galera_port;evs.auto_evict=1'
+
+[mysqld.4]
+wsrep_provider_options='base_port=@mysqld.4.#galera_port;evs.auto_evict=1'

mysql-test/suite/galera/t/galera_event_node_evict.test

@@ -0,0 +1,96 @@
+#
+# Test that node eviction event is generated in the status file.
+#
+
+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+# Make sure that Galera library has node eviction event reporting.
+--let $galera_version=26.4.14
+--source suite/wsrep/include/check_galera_version.inc
+
+--connection node_1
+CALL mtr.add_suppression("\\[Warning\\] WSREP: evicting member .* at .* permanently from group");
+
+--connection node_2
+CALL mtr.add_suppression("\\[Warning\\] WSREP: handshake with .* .* failed: 'evicted'");
+CALL mtr.add_suppression("\\[ERROR\\] WSREP: exception from gcomm, backend must be restarted: this node has been evicted out of the cluster, gcomm backend restart is required \\(FATAL\\)");
+
+--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
+CALL mtr.add_suppression("\\[Warning\\] WSREP: evicting member .* at .* permanently from group");
+
+--connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4
+CALL mtr.add_suppression("\\[Warning\\] WSREP: evicting member .* at .* permanently from group");
+
+# Save original auto_increment_offset values.
+
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--let $node_4=node_4
+--source include/auto_increment_offset_save.inc
+
+# Repeatedly isolate the node from the cluster until it's evicted.
+
+--disable_query_log
+--disable_result_log
+
+--let $evicted = 0
+while (!$evicted)
+{
+    --connection node_2
+
+    --let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+    --source include/wait_condition.inc
+
+    --let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+    --source include/wait_condition.inc
+
+    --disable_query_log
+    --disable_result_log
+
+    SET GLOBAL wsrep_provider_options='gmcast.isolate=1';
+
+    --connection node_1
+    --let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+    --source include/wait_condition.inc
+
+    --disable_query_log
+    --disable_result_log
+
+    --connection node_2
+    SET GLOBAL wsrep_provider_options='gmcast.isolate=0';
+
+    # As the cluster size doesn't change in case the node is evicted, there is
+    # no good condition to wait for in case the eviction happens, so sleep a bit.
+
+    --sleep 1
+
+    --connection node_1
+    --let $evicted = `SELECT VARIABLE_VALUE != '' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_evs_evict_list'`
+}
+
+--enable_query_log
+--enable_result_log
+
+--connection node_2
+--source include/wsrep_wait_disconnect.inc
+SET GLOBAL wsrep_on = OFF;
+
+# Check the status file to contain the eviction event message.
+
+--let $assert_count = 1
+--let $assert_file = $MYSQLTEST_VARDIR/mysqld.2/data/status2.json
+--let $assert_text = Node evicted
+--let $assert_select = "status": "evicted"
+--source include/assert_grep.inc
+
+# Re-join the node after restart.
+
+--source include/restart_mysqld.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+--source include/auto_increment_offset_restore.inc

sql/CMakeLists.txt

@@ -22,6 +22,7 @@ IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY)
     wsrep_server_service.cc
     wsrep_storage_service.cc
     wsrep_server_state.cc
+    wsrep_event_service.cc
     wsrep_status.cc
     wsrep_allowlist_service.cc
     wsrep_utils.cc

sql/wsrep_event_service.cc

@@ -0,0 +1,23 @@
+/* Copyright 2021-2022 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include "wsrep_event_service.h"
+
+wsrep::event_service*
+Wsrep_event_service::instance()
+{
+  static Wsrep_event_service instance;
+  return &instance;
+}

sql/wsrep_event_service.h

@@ -0,0 +1,49 @@
+/* Copyright 2021-2022 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef WSREP_EVENT_SERVICE_H
+#define WSREP_EVENT_SERVICE_H
+
+/* wsrep-lib */
+#include "wsrep/event_service.hpp"
+
+/* implementation */
+#include "wsrep_status.h"
+
+class Wsrep_event_service : public wsrep::event_service
+{
+public:
+
+  void process_event(const std::string& name, const std::string& value)
+    override
+  {
+    if (name == "progress")
+    {
+      Wsrep_status::report_progress(value);
+    }
+    else if (name == "event")
+    {
+      Wsrep_status::report_event(value);
+    }
+    else
+    {
+      // not interested in the event
+    }
+  }
+
+  static wsrep::event_service* instance();
+};
+
+#endif /* WSREP_EVENT_SERVICE_H */

sql/wsrep_server_state.cc

@@ -17,6 +17,7 @@
 #include "wsrep_api.h"
 #include "wsrep_server_state.h"
 #include "wsrep_allowlist_service.h"
+#include "wsrep_event_service.h"
 #include "wsrep_binlog.h" /* init/deinit group commit */
 
 mysql_mutex_t LOCK_wsrep_server_state;
@@ -89,6 +90,7 @@ void Wsrep_server_state::destroy()
 void Wsrep_server_state::init_provider_services()
 {
   m_provider_services.allowlist_service= wsrep_allowlist_service_init();
+  m_provider_services.event_service= Wsrep_event_service::instance();
 }
 
 void Wsrep_server_state::deinit_provider_services()

sql/wsrep_status.h

@@ -42,6 +42,13 @@ public:
     Wsrep_status::m_instance->report_progress(progress);
   }
 
+  static void report_event(const std::string& event)
+  {
+    if (!Wsrep_status::m_instance) return;
+
+    Wsrep_status::m_instance->report_event(event);
+  }
+
   static void report_log_msg(wsrep::reporter::log_level level,
                              const char* tag, size_t tag_len,
                              const char* buf, size_t buf_len,