From c872125a664842ecfb66c60f69b3a87390aec23d Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Sat, 22 May 2021 15:53:33 +0300
Subject: [PATCH 01/25] MDEV-25630: Crash with window function in left expr of
 IN subquery

* Make Item_in_optimizer::fix_fields inherit the with_window_func
  attribute of the subquery's left expression (the subquery itself
  cannot have window functions that are aggregated in this select)

* Make Item_cache_wrapper::Item_cache_wrapper() inherit
  with_window_func attribute of the item it is caching.
---
 mysql-test/r/win.result                       | 19 +++++++++++++++++++
 .../encryption/r/tempfiles_encrypted.result   | 19 +++++++++++++++++++
 mysql-test/t/win.test                         | 14 ++++++++++++++
 sql/item.cc                                   |  1 +
 sql/item_cmpfunc.cc                           |  3 +++
 5 files changed, 56 insertions(+)

diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result
index dd74c5c77fd..8a31dcc0634 100644
--- a/mysql-test/r/win.result
+++ b/mysql-test/r/win.result
@@ -3892,5 +3892,24 @@ id	rn
 1	1
 drop table t1;
 #
+# MDEV-25630: Crash with window function in left expr of IN subquery
+#
+CREATE TABLE t1 (i int);
+INSERT INTO t1 VALUES (1),(2),(3);
+SELECT lag(i) over (ORDER BY i) IN ( SELECT 1 FROM t1 a) FROM t1;
+lag(i) over (ORDER BY i) IN ( SELECT 1 FROM t1 a)
+NULL
+1
+0
+DROP TABLE t1;
+CREATE TABLE t1 (i int);
+INSERT INTO t1 VALUES (1),(2),(3);
+SELECT sum(i) over () IN ( SELECT 1 FROM t1 a) FROM t1;
+sum(i) over () IN ( SELECT 1 FROM t1 a)
+0
+0
+0
+DROP TABLE t1;
+#
 # End of 10.2 tests
 #
diff --git a/mysql-test/suite/encryption/r/tempfiles_encrypted.result b/mysql-test/suite/encryption/r/tempfiles_encrypted.result
index 27eedc45028..3c81d7b6046 100644
--- a/mysql-test/suite/encryption/r/tempfiles_encrypted.result
+++ b/mysql-test/suite/encryption/r/tempfiles_encrypted.result
@@ -3898,6 +3898,25 @@ id	rn
 1	1
 drop table t1;
 #
+# MDEV-25630: Crash with window function in left expr of IN subquery
+#
+CREATE TABLE t1 (i int);
+INSERT INTO t1 VALUES (1),(2),(3);
+SELECT lag(i) over (ORDER BY i) IN ( SELECT 1 FROM t1 a) FROM t1;
+lag(i) over (ORDER BY i) IN ( SELECT 1 FROM t1 a)
+NULL
+1
+0
+DROP TABLE t1;
+CREATE TABLE t1 (i int);
+INSERT INTO t1 VALUES (1),(2),(3);
+SELECT sum(i) over () IN ( SELECT 1 FROM t1 a) FROM t1;
+sum(i) over () IN ( SELECT 1 FROM t1 a)
+0
+0
+0
+DROP TABLE t1;
+#
 # End of 10.2 tests
 #
 #
diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test
index 57214ab0165..c07a81f17da 100644
--- a/mysql-test/t/win.test
+++ b/mysql-test/t/win.test
@@ -2542,6 +2542,20 @@ order by rn desc;
 
 drop table t1;
 
+--echo #
+--echo # MDEV-25630: Crash with window function in left expr of IN subquery
+--echo #
+
+CREATE TABLE t1 (i int);
+INSERT INTO t1 VALUES (1),(2),(3);
+SELECT lag(i) over (ORDER BY i) IN ( SELECT 1 FROM t1 a) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (i int);
+INSERT INTO t1 VALUES (1),(2),(3);
+SELECT sum(i) over () IN ( SELECT 1 FROM t1 a) FROM t1;
+DROP TABLE t1;
+
 --echo #
 --echo # End of 10.2 tests
 --echo #
diff --git a/sql/item.cc b/sql/item.cc
index be64edca9a1..d7a3659a2ce 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -8203,6 +8203,7 @@ Item_cache_wrapper::Item_cache_wrapper(THD *thd, Item *item_arg):
   name= item_arg->name;
   name_length= item_arg->name_length;
   with_subselect=  orig_item->with_subselect;
+  with_window_func= orig_item->with_window_func;
 
   if ((expr_value= Item_cache::get_cache(thd, orig_item)))
     expr_value->setup(thd, orig_item);
diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index 7b7604053e3..8a2c532f621 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -1416,6 +1416,9 @@ bool Item_in_optimizer::fix_fields(THD *thd, Item **ref)
     maybe_null=1;
   with_subselect= 1;
   with_sum_func= with_sum_func || args[1]->with_sum_func;
+  with_window_func= args[0]->with_window_func;
+  // The subquery cannot have window functions aggregated in this select
+  DBUG_ASSERT(!args[1]->with_window_func);
   with_field= with_field || args[1]->with_field;
   with_param= args[0]->with_param || args[1]->with_param; 
   used_tables_and_const_cache_join(args[1]);

From 7a1eff0a9d90a45e774ca126ab10f86a78b930f4 Mon Sep 17 00:00:00 2001
From: Elena Stepanova <elenst@mariadb.com>
Date: Wed, 9 Jun 2021 15:28:35 +0300
Subject: [PATCH 02/25] MDEV-25884 Tests use environment $USER variable without
 quotes

---
 mysql-test/suite/mariabackup/auth_plugin_win.result |  2 +-
 mysql-test/suite/mariabackup/auth_plugin_win.test   |  2 +-
 mysql-test/suite/plugins/r/unix_socket.result       |  4 ++--
 mysql-test/suite/plugins/t/unix_socket.test         | 12 ++++++------
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/mysql-test/suite/mariabackup/auth_plugin_win.result b/mysql-test/suite/mariabackup/auth_plugin_win.result
index 7a623be147f..caf5d8df87d 100644
--- a/mysql-test/suite/mariabackup/auth_plugin_win.result
+++ b/mysql-test/suite/mariabackup/auth_plugin_win.result
@@ -1,5 +1,5 @@
 INSTALL SONAME 'auth_named_pipe';
 CREATE USER 'USERNAME' IDENTIFIED WITH named_pipe;
-GRANT ALL PRIVILEGES ON *.* to USERNAME;
+GRANT ALL PRIVILEGES ON *.* to 'USERNAME';
 DROP USER 'USERNAME';
 UNINSTALL SONAME 'auth_named_pipe';
diff --git a/mysql-test/suite/mariabackup/auth_plugin_win.test b/mysql-test/suite/mariabackup/auth_plugin_win.test
index 9c8cd5ad411..70ae74b7028 100644
--- a/mysql-test/suite/mariabackup/auth_plugin_win.test
+++ b/mysql-test/suite/mariabackup/auth_plugin_win.test
@@ -18,7 +18,7 @@ INSTALL SONAME 'auth_named_pipe';
 --replace_result $USERNAME USERNAME
 eval CREATE USER '$USERNAME' IDENTIFIED WITH named_pipe;
 --replace_result $USERNAME USERNAME
-eval GRANT ALL PRIVILEGES ON *.* to $USERNAME;
+eval GRANT ALL PRIVILEGES ON *.* to '$USERNAME';
 
 let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
 --disable_result_log
diff --git a/mysql-test/suite/plugins/r/unix_socket.result b/mysql-test/suite/plugins/r/unix_socket.result
index 0e08794fbe6..096b736cd68 100644
--- a/mysql-test/suite/plugins/r/unix_socket.result
+++ b/mysql-test/suite/plugins/r/unix_socket.result
@@ -2,7 +2,7 @@ install plugin unix_socket soname 'auth_socket.so';
 #
 # with named user
 #
-create user USER identified via unix_socket;
+create user 'USER' identified via unix_socket;
 #
 # name match = ok
 #
@@ -12,7 +12,7 @@ USER@localhost	USER@%	test
 #
 # name does not match = failure
 #
-drop user USER;
+drop user 'USER';
 #
 # and now with anonymous user
 #
diff --git a/mysql-test/suite/plugins/t/unix_socket.test b/mysql-test/suite/plugins/t/unix_socket.test
index bd0323c0274..75409ec70b4 100644
--- a/mysql-test/suite/plugins/t/unix_socket.test
+++ b/mysql-test/suite/plugins/t/unix_socket.test
@@ -12,9 +12,9 @@ eval install plugin unix_socket soname '$AUTH_SOCKET_SO';
 --echo # with named user
 --echo #
 
---let $replace=create user $USER
---replace_result $replace "create user USER"
-eval create user $USER identified via unix_socket;
+--let $replace=create user '$USER'
+--replace_result $replace "create user 'USER'"
+eval create user '$USER' identified via unix_socket;
 
 --write_file $MYSQLTEST_VARDIR/tmp/peercred_test.txt
 --let $replace1=$USER@localhost
@@ -34,9 +34,9 @@ EOF
 --error 1
 --exec $MYSQL_TEST -u foobar --plugin-dir=$plugindir < $MYSQLTEST_VARDIR/tmp/peercred_test.txt
 
---let $replace=drop user $USER
---replace_result $replace "drop user USER"
-eval drop user $USER;
+--let $replace=drop user '$USER'
+--replace_result $replace "drop user 'USER'"
+eval drop user '$USER';
 
 --echo #
 --echo # and now with anonymous user

From 396864c6b3d2cca9962f7eb58964f50ed6b612dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 10 Jun 2021 11:17:40 +0300
Subject: [PATCH 03/25] Remove orphan type name trx_sig_t

This was missed in commit 1d0f70c2f894b27e98773a282871d32802f67964.
---
 storage/innobase/include/trx0types.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/storage/innobase/include/trx0types.h b/storage/innobase/include/trx0types.h
index 097aea519a9..26589bb2a96 100644
--- a/storage/innobase/include/trx0types.h
+++ b/storage/innobase/include/trx0types.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2014, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2019, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -96,8 +96,6 @@ struct trx_t;
 struct trx_lock_t;
 /** Transaction system */
 struct trx_sys_t;
-/** Signal */
-struct trx_sig_t;
 /** Rollback segment */
 struct trx_rseg_t;
 /** Transaction undo log */

From 152c83d49ca821c54aa49f6b43e33cba63e4d19f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= <otto@kekalainen.net>
Date: Sun, 30 May 2021 15:42:04 -0700
Subject: [PATCH 04/25] MDEV-20392: Skip ABI check if 'diff' is not found

Not all environments have 'diff' installed. Most notably CentOS 8
does not have diff out-of-the-box. Thus users running 'cmake .' and
'make' would fail to build MariaDB, and they would think the error
was in ABI incompatibilities due to the error message emitted by CMake
when in reality simply 'diff' was missing.

This fixes it and makes the developer experience better by simply
skipping the diffing if 'diff' is not found.

Closes #1846
---
 cmake/do_abi_check.cmake | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cmake/do_abi_check.cmake b/cmake/do_abi_check.cmake
index 43d8b15a7ab..0ad0fa39670 100644
--- a/cmake/do_abi_check.cmake
+++ b/cmake/do_abi_check.cmake
@@ -74,7 +74,9 @@ FOREACH(file ${ABI_HEADERS})
   FILE(REMOVE ${tmpfile})
   EXECUTE_PROCESS(
     COMMAND diff -w ${file}.pp ${abi_check_out} RESULT_VARIABLE result)
-  IF(NOT ${result} EQUAL 0)
+  IF(result MATCHES "No such file or directory")
+    MESSAGE("Command 'diff' not found. ABI check for ${file} skipped.")
+  ELSEIF(NOT result EQUAL 0)
     IF(ABI_UPDATE)
       EXECUTE_PROCESS(COMMAND mv -v ${abi_check_out} ${file}.pp)
     ELSE(ABI_UPDATE)
@@ -84,4 +86,3 @@ FOREACH(file ${ABI_HEADERS})
   ENDIF()
   FILE(REMOVE ${abi_check_out})
 ENDFOREACH()
-

From 4352c77c5a3ac89acc5fd90a38f806d0ec500aa4 Mon Sep 17 00:00:00 2001
From: Brad Smith <brad@comstyle.com>
Date: Thu, 10 Jun 2021 20:01:44 -0400
Subject: [PATCH 05/25] Link with libexecinfo on OpenBSD for stacktrace
 functionality.

---
 cmake/os/OpenBSD.cmake | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 cmake/os/OpenBSD.cmake

diff --git a/cmake/os/OpenBSD.cmake b/cmake/os/OpenBSD.cmake
new file mode 100644
index 00000000000..a96cc891f38
--- /dev/null
+++ b/cmake/os/OpenBSD.cmake
@@ -0,0 +1,23 @@
+
+# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1335  USA 
+
+# This file includes OpenBSD specific options and quirks, related to system checks
+
+# Find libexecinfo (library that contains backtrace_symbols etc)
+FIND_LIBRARY(EXECINFO NAMES execinfo)
+IF(EXECINFO)
+ SET(LIBEXECINFO ${EXECINFO})
+ENDIF()

From 887f46a618ac2351dbf9f391f13aca9ba6fa9c54 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Sat, 12 Jun 2021 12:23:07 +0200
Subject: [PATCH 06/25] fix mysqltest crash report output

* read_command_buf is a pointer now, sizeof() no longer reflects its
  length, read_command_buflen is.
* my_safe_print_str() prints multiple screens of '\0' bytes after the
  query end and up to read_command_buflen. Use fprintf() instead.
* when setting connection->name to "-closed_connection-" update
  connection->name_len to match.
---
 client/mysqltest.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index 2a9560e8903..1587d7513b3 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -90,6 +90,8 @@ static my_bool non_blocking_api_enabled= 0;
 
 #define QUERY_PRINT_ORIGINAL_FLAG 4
 
+#define CLOSED_CONNECTION "-closed_connection-"
+
 #ifndef HAVE_SETENV
 static int setenv(const char *name, const char *value, int overwrite);
 #endif
@@ -5569,11 +5571,12 @@ void do_close_connection(struct st_command *command)
   my_free(con->name);
 
   /*
-    When the connection is closed set name to "-closed_connection-"
+    When the connection is closed set name to CLOSED_CONNECTION
     to make it possible to reuse the connection name.
   */
-  if (!(con->name = my_strdup("-closed_connection-", MYF(MY_WME))))
+  if (!(con->name = my_strdup(CLOSED_CONNECTION, MYF(MY_WME))))
     die("Out of memory");
+  con->name_len= sizeof(CLOSED_CONNECTION)-1;
 
   if (con == cur_con)
   {
@@ -5957,7 +5960,7 @@ void do_connect(struct st_command *command)
     con_slot= next_con;
   else
   {
-    if (!(con_slot= find_connection_by_name("-closed_connection-")))
+    if (!(con_slot= find_connection_by_name(CLOSED_CONNECTION)))
       die("Connection limit exhausted, you can have max %d connections",
           opt_max_connections);
     my_free(con_slot->name);
@@ -8980,7 +8983,7 @@ static void dump_backtrace(void)
   struct st_connection *conn= cur_con;
 
   fprintf(stderr, "read_command_buf (%p): ", read_command_buf);
-  my_safe_print_str(read_command_buf, sizeof(read_command_buf));
+  fprintf(stderr, "%.*s\n", (int)read_command_buflen, read_command_buf);
   fputc('\n', stderr);
 
   if (conn)

From c9f9e38bb52cea32ffef7585dfa089a407908a30 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Sat, 12 Jun 2021 12:38:45 +0200
Subject: [PATCH 07/25] fix mysqlest crash on ./mtr --sp
 innodb_fts.innodb-fts-stopword

---
 client/mysqltest.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index 1587d7513b3..6d5b6ff31b6 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -8574,7 +8574,7 @@ void run_query(struct st_connection *cn, struct st_command *command, int flags)
   log_file.flush();
   dynstr_set(&ds_res, 0);
 
-  if (view_protocol_enabled &&
+  if (view_protocol_enabled && mysql &&
       complete_query &&
       match_re(&view_re, query))
   {
@@ -8620,7 +8620,7 @@ void run_query(struct st_connection *cn, struct st_command *command, int flags)
     dynstr_free(&query_str);
   }
 
-  if (sp_protocol_enabled &&
+  if (sp_protocol_enabled && mysql &&
       complete_query &&
       match_re(&sp_re, query))
   {

From 2e33f574b37c7544d5b4d1c0e079cf65e29a1346 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 14 Jun 2021 23:45:31 +0200
Subject: [PATCH 08/25] update libmariadb

---
 libmariadb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libmariadb b/libmariadb
index 180c543704d..802ce584a26 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit 180c543704d627a50a52aaf60e24ca14e0ec4686
+Subproject commit 802ce584a26fdc0ba67fcf35e277bf3c7440956a

From ec4df51414d34aff18c8473386b85c9ea53c44e0 Mon Sep 17 00:00:00 2001
From: Jordy Zomer <jordy@pwning.systems>
Date: Tue, 15 Jun 2021 12:34:23 +1000
Subject: [PATCH 09/25] eventscheduler mismatch of my_{malloc,free}, delete

Fix malloc/delete mismatch. This causes a double free in the cleanup.

closes #1845
---
 sql/event_scheduler.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/event_scheduler.cc b/sql/event_scheduler.cc
index 2a5399fb94a..423f7e6ea2b 100644
--- a/sql/event_scheduler.cc
+++ b/sql/event_scheduler.cc
@@ -436,7 +436,7 @@ Event_scheduler::start(int *err_no)
     scheduler_thd= NULL;
     deinit_event_thread(new_thd);
 
-    delete scheduler_param_value;
+    my_free(scheduler_param_value);
     ret= true;
   }
 

From be243ed9e3e33e3678300c20ab4e5f75c37a8f98 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Tue, 15 Jun 2021 12:40:33 +1000
Subject: [PATCH 10/25] cmake OpenBSD copyright notice correction

Brad Smith made this OpenBSD file based of the
FreeBSD cmake directives in commit ab589043670145c95ff372021bab19464b6036e2
by the Monty Program Ab.

As such the Oracle Copyright header isn't really applicable.
---
 cmake/os/OpenBSD.cmake | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cmake/os/OpenBSD.cmake b/cmake/os/OpenBSD.cmake
index a96cc891f38..c8b91944275 100644
--- a/cmake/os/OpenBSD.cmake
+++ b/cmake/os/OpenBSD.cmake
@@ -1,5 +1,4 @@
-
-# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright (C) 2012 Monty Program Ab, 2021 Brad Smith
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

From 7d2c338c48a9235c5c5e76293760cabc2ff48723 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Mon, 14 Jun 2021 19:37:06 +0530
Subject: [PATCH 11/25] MDEV-14180 preparation: Rename key_rotation_list

- Rename the rotation list to default_encrypt_tables in
fil_system_t. Because rotation list naming could be
misleading when it comes to key version rotation

- Rename is_in_rotation_list to is_in_default_encrypt
in fil_space_t

- Rename keyrotate_next function to default_encrypt_next

fil_system_t::default_encrypt_next(): Find the next
suitable default encrypt table if beginning of
default_encrypt_tables list has been scheduled
to be deleted
---
 storage/innobase/fil/fil0crypt.cc  | 55 ++++++++++++++++++------------
 storage/innobase/fil/fil0fil.cc    | 11 +++---
 storage/innobase/include/fil0fil.h | 14 ++++----
 3 files changed, 46 insertions(+), 34 deletions(-)

diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 68a8a9be261..b167a589695 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -1380,52 +1380,65 @@ fil_crypt_return_iops(
 	fil_crypt_update_total_stat(state);
 }
 
-/** Return the next tablespace from rotation_list.
+/** Return the next tablespace from default_encrypt_tables.
 @param space   previous tablespace (NULL to start from the start)
 @param recheck whether the removal condition needs to be rechecked after
 the encryption parameters were changed
 @param encrypt expected state of innodb_encrypt_tables
 @return the next tablespace to process (n_pending_ops incremented)
 @retval NULL if this was the last */
-inline fil_space_t *fil_system_t::keyrotate_next(fil_space_t *space,
-                                                 bool recheck, bool encrypt)
+inline fil_space_t *fil_system_t::default_encrypt_next(
+  fil_space_t *space, bool recheck, bool encrypt)
 {
   ut_ad(mutex_own(&mutex));
 
   sized_ilist<fil_space_t, rotation_list_tag_t>::iterator it=
-    space && space->is_in_rotation_list ? space : rotation_list.begin();
+    space && space->is_in_default_encrypt
+    ? space
+    : default_encrypt_tables.begin();
   const sized_ilist<fil_space_t, rotation_list_tag_t>::iterator end=
-    rotation_list.end();
+    default_encrypt_tables.end();
 
   if (space)
   {
     const bool released= !--space->n_pending_ops;
 
-    if (space->is_in_rotation_list)
+    if (space->is_in_default_encrypt)
     {
       while (++it != end &&
              (!UT_LIST_GET_LEN(it->chain) || it->is_stopping()));
 
-      /* If one of the encryption threads already started the encryption
-      of the table then don't remove the unencrypted spaces from rotation list
+      /* If one of the encryption threads already started
+      the encryption of the table then don't remove the
+      unencrypted spaces from default encrypt list.
 
-      If there is a change in innodb_encrypt_tables variables value then
-      don't remove the last processed tablespace from the rotation list. */
+      If there is a change in innodb_encrypt_tables variables
+      value then don't remove the last processed tablespace
+      from the default encrypt list. */
       if (released && (!recheck || space->crypt_data) &&
           !encrypt == !srv_encrypt_tables)
       {
-        ut_a(!rotation_list.empty());
-        rotation_list.remove(*space);
-        space->is_in_rotation_list= false;
+        ut_a(!default_encrypt_tables.empty());
+        default_encrypt_tables.remove(*space);
+        space->is_in_default_encrypt= false;
       }
     }
   }
+  else while (it != end &&
+	      (!UT_LIST_GET_LEN(it->chain) || it->is_stopping()))
+  {
+    /* Find the next suitable default encrypt table if
+    beginning of default_encrypt_tables list has been scheduled
+    to be deleted */
+    it++;
+  }
 
   if (it == end)
     return NULL;
 
   space= &*it;
   space->n_pending_ops++;
+  ut_ad(!space->is_stopping());
   return space;
 }
 
@@ -1443,7 +1456,7 @@ static fil_space_t *fil_space_next(fil_space_t *space, bool recheck,
   ut_ad(!space || space->n_pending_ops);
 
   if (!srv_fil_crypt_rotate_key_age)
-    space= fil_system->keyrotate_next(space, recheck, encrypt);
+    space= fil_system->default_encrypt_next(space, recheck, encrypt);
   else if (!space)
   {
     space= UT_LIST_GET_FIRST(fil_system->space_list);
@@ -2335,9 +2348,9 @@ fil_crypt_set_thread_cnt(
 	}
 }
 
-/** Initialize the tablespace rotation_list
+/** Initialize the tablespace default_encrypt_tables
 if innodb_encryption_rotate_key_age=0. */
-static void fil_crypt_rotation_list_fill()
+static void fil_crypt_default_encrypt_tables_fill()
 {
 	ut_ad(mutex_own(&fil_system->mutex));
 
@@ -2345,7 +2358,7 @@ static void fil_crypt_rotation_list_fill()
 	     space != NULL;
 	     space = UT_LIST_GET_NEXT(space_list, space)) {
 		if (space->purpose != FIL_TYPE_TABLESPACE
-		    || space->is_in_rotation_list
+		    || space->is_in_default_encrypt
 		    || space->is_stopping()
 		    || UT_LIST_GET_LEN(space->chain) == 0) {
 			continue;
@@ -2389,8 +2402,8 @@ static void fil_crypt_rotation_list_fill()
 			}
 		}
 
-		fil_system->rotation_list.push_back(*space);
-		space->is_in_rotation_list = true;
+		fil_system->default_encrypt_tables.push_back(*space);
+		space->is_in_default_encrypt = true;
 	}
 }
 
@@ -2405,7 +2418,7 @@ fil_crypt_set_rotate_key_age(
 	mutex_enter(&fil_system->mutex);
 	srv_fil_crypt_rotate_key_age = val;
 	if (val == 0) {
-		fil_crypt_rotation_list_fill();
+		fil_crypt_default_encrypt_tables_fill();
 	}
 	mutex_exit(&fil_system->mutex);
 	os_event_set(fil_crypt_threads_event);
@@ -2436,7 +2449,7 @@ fil_crypt_set_encrypt_tables(
 	srv_encrypt_tables = val;
 
 	if (srv_fil_crypt_rotate_key_age == 0) {
-		fil_crypt_rotation_list_fill();
+		fil_crypt_default_encrypt_tables_fill();
 	}
 
 	mutex_exit(&fil_system->mutex);
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index 2607856bc74..3dc804ea878 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -1244,10 +1244,9 @@ fil_space_detach(
 		space->is_in_unflushed_spaces = false;
 	}
 
-	if (space->is_in_rotation_list) {
-
-		fil_system->rotation_list.remove(*space);
-		space->is_in_rotation_list = false;
+	if (space->is_in_default_encrypt) {
+		fil_system->default_encrypt_tables.remove(*space);
+		space->is_in_default_encrypt = false;
 	}
 
 	UT_LIST_REMOVE(fil_system->space_list, space);
@@ -1473,8 +1472,8 @@ fil_space_create(
 		    srv_encrypt_tables)) {
 		/* Key rotation is not enabled, need to inform background
 		encryption threads. */
-		fil_system->rotation_list.push_back(*space);
-		space->is_in_rotation_list = true;
+		fil_system->default_encrypt_tables.push_back(*space);
+		space->is_in_default_encrypt = true;
 		mutex_exit(&fil_system->mutex);
 		mutex_enter(&fil_crypt_threads_mutex);
 		os_event_set(fil_crypt_threads_event);
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index 65457b414ca..7ce89f3be60 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -177,7 +177,7 @@ struct fil_space_t : ilist_node<unflushed_spaces_tag_t>,
 	bool is_in_unflushed_spaces;
 
 	/** Checks that this tablespace needs key rotation. */
-	bool is_in_rotation_list;
+	bool is_in_default_encrypt;
 
 	/** True if the device this filespace is on supports atomic writes */
 	bool		atomic_write_supported;
@@ -541,9 +541,9 @@ struct fil_system_t {
 					record has been written since
 					the latest redo log checkpoint.
 					Protected only by log_sys->mutex. */
-	ilist<fil_space_t, rotation_list_tag_t> rotation_list;
-					/*!< list of all file spaces needing
-					key rotation.*/
+
+	/** List of all file spaces need key rotation */
+	ilist<fil_space_t, rotation_list_tag_t> default_encrypt_tables;
 
 	bool		space_id_reuse_warned;
 					/* !< TRUE if fil_space_create()
@@ -556,15 +556,15 @@ struct fil_system_t {
 	@retval	NULL	if the tablespace does not exist or cannot be read */
 	fil_space_t* read_page0(ulint id);
 
-  /** Return the next tablespace from rotation_list.
+  /** Return the next tablespace from default_encrypt_tables list.
   @param space   previous tablespace (NULL to start from the start)
   @param recheck whether the removal condition needs to be rechecked after
   the encryption parameters were changed
   @param encrypt expected state of innodb_encrypt_tables
   @return the next tablespace to process (n_pending_ops incremented)
   @retval NULL if this was the last */
-  inline fil_space_t* keyrotate_next(fil_space_t *space, bool recheck,
-                                     bool encrypt);
+  inline fil_space_t* default_encrypt_next(
+    fil_space_t *space, bool recheck, bool encrypt);
 };
 
 /** The tablespace memory cache. This variable is NULL before the module is

From 8c7d8b716c976d28b3373510543fa88f6d12a643 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Tue, 15 Jun 2021 13:12:15 +0530
Subject: [PATCH 12/25] MDEV-14180 Automatically disable key rotation checks
 for file_key_managment plugin

Problem:
=======
- InnoDB iterates the fil_system space list to encrypt the
tablespace in case of key rotation. But it is not
necessary for any encryption plugin which doesn't do
key version rotation.

Solution:
=========
- Introduce a new variable called srv_encrypt_rotate to
indicate whether encryption plugin does key rotation

fil_space_crypt_t::key_get_latest_version(): Enable the
srv_encrypt_rotate only once if current key version is
higher than innodb_encyrption_rotate_key_age

fil_crypt_must_default_encrypt(): Default encryption tables
should be added to default_encryp_tables list if
innodb_encyrption_rotate_key_age is zero and encryption
plugin doesn't do key version rotation

fil_space_create(): Add the newly created space to
default_encrypt_tables list if
fil_crypt_must_default_encrypt() returns true

Removed the nondeterministic select from
innodb-key-rotation-disable test. By default,
InnoDB adds the tablespace to the rotation list and
background crypt thread does encryption of tablespace.
So these select doesn't give reliable results.
---
 .../r/innodb-key-rotation-disable.result      |  4 --
 .../encryption/r/key_version_rotation.result  | 19 +++++++++
 .../t/innodb-key-rotation-disable.test        |  3 --
 .../encryption/t/key_version_rotation.opt     |  2 +
 .../encryption/t/key_version_rotation.test    | 41 +++++++++++++++++++
 storage/innobase/fil/fil0crypt.cc             | 20 ++++++++-
 storage/innobase/fil/fil0fil.cc               | 18 +++++---
 storage/innobase/include/fil0crypt.h          |  6 +++
 8 files changed, 98 insertions(+), 15 deletions(-)
 create mode 100644 mysql-test/suite/encryption/r/key_version_rotation.result
 create mode 100644 mysql-test/suite/encryption/t/key_version_rotation.opt
 create mode 100644 mysql-test/suite/encryption/t/key_version_rotation.test

diff --git a/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result b/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result
index a662f5e6343..7d0267d5057 100644
--- a/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result
+++ b/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result
@@ -1,7 +1,3 @@
-SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
-NAME
-SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
-NAME
 create database enctests;
 use enctests;
 create table t1(a int not null primary key, b char(200)) engine=innodb;
diff --git a/mysql-test/suite/encryption/r/key_version_rotation.result b/mysql-test/suite/encryption/r/key_version_rotation.result
new file mode 100644
index 00000000000..1a295abfe8d
--- /dev/null
+++ b/mysql-test/suite/encryption/r/key_version_rotation.result
@@ -0,0 +1,19 @@
+create table t1(f1 int not null)engine=innodb;
+create table t2(f1 int not null)engine=innodb;
+insert into t1 select * from seq_1_to_100;
+insert into t2 select * from seq_1_to_100;
+# Enable encryption
+set global innodb_encrypt_tables=ON;
+# Create a new table and it is added to rotation list
+create table t3(f1 int not null)engine=innodb;
+insert into t3 select * from seq_1_to_100;
+# Increase the version and it should set rotation
+# variable for the encryption plugin
+set global debug_key_management_version=10;
+select  @@debug_key_management_version;
+@@debug_key_management_version
+10
+# Decrease the key version and Disable the encryption
+set global debug_key_management_version=1;
+set global innodb_encrypt_tables=off;
+DROP TABLE t1, t2, t3;
diff --git a/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test b/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test
index 2e5d877f7c0..4541bfa2a0c 100644
--- a/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test
+++ b/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test
@@ -3,9 +3,6 @@
 # not embedded because of restarts
 -- source include/not_embedded.inc
 
-SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
-SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
-
 --disable_query_log
 --disable_warnings
 let $encryption = `SELECT @@innodb_encrypt_tables`;
diff --git a/mysql-test/suite/encryption/t/key_version_rotation.opt b/mysql-test/suite/encryption/t/key_version_rotation.opt
new file mode 100644
index 00000000000..d7933f0f943
--- /dev/null
+++ b/mysql-test/suite/encryption/t/key_version_rotation.opt
@@ -0,0 +1,2 @@
+--innodb-tablespaces-encryption
+--plugin-load-add=$DEBUG_KEY_MANAGEMENT_SO
diff --git a/mysql-test/suite/encryption/t/key_version_rotation.test b/mysql-test/suite/encryption/t/key_version_rotation.test
new file mode 100644
index 00000000000..d36d47251a1
--- /dev/null
+++ b/mysql-test/suite/encryption/t/key_version_rotation.test
@@ -0,0 +1,41 @@
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_sequence.inc
+
+create table t1(f1 int not null)engine=innodb;
+create table t2(f1 int not null)engine=innodb;
+insert into t1 select * from seq_1_to_100;
+insert into t2 select * from seq_1_to_100;
+
+let $restart_parameters=--innodb_encrypt_tables=0 --innodb_encryption_threads=1 --innodb_encryption_rotate_key_age=9;
+--source include/restart_mysqld.inc
+
+--echo # Enable encryption
+
+set global innodb_encrypt_tables=ON;
+--let $tables_count= `select count(*) from information_schema.tables where engine = 'InnoDB'`
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--echo # Create a new table and it is added to rotation list
+create table t3(f1 int not null)engine=innodb;
+insert into t3 select * from seq_1_to_100;
+
+--echo # Increase the version and it should set rotation
+--echo # variable for the encryption plugin
+
+set global debug_key_management_version=10;
+select  @@debug_key_management_version;
+--let $tables_count= `select count(*) from information_schema.tables where engine = 'InnoDB'`
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+
+--echo # Decrease the key version and Disable the encryption
+set global debug_key_management_version=1;
+set global innodb_encrypt_tables=off;
+
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+--source include/wait_condition.inc
+DROP TABLE t1, t2, t3;
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index b167a589695..8195716f722 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -55,6 +55,9 @@ UNIV_INTERN uint srv_n_fil_crypt_threads_started = 0;
 /** At this age or older a space/page will be rotated */
 UNIV_INTERN uint srv_fil_crypt_rotate_key_age;
 
+/** Whether the encryption plugin does key rotation */
+static bool srv_encrypt_rotate;
+
 /** Event to signal FROM the key rotation threads. */
 static os_event_t fil_crypt_event;
 
@@ -136,6 +139,14 @@ fil_space_crypt_t::key_get_latest_version(void)
 
 	if (is_key_found()) {
 		key_version = encryption_key_get_latest_version(key_id);
+		/* InnoDB does dirty read of srv_fil_crypt_rotate_key_age.
+		It doesn't matter because srv_encrypt_rotate
+		can be set to true only once */
+		if (!srv_encrypt_rotate
+		    && key_version > srv_fil_crypt_rotate_key_age) {
+			srv_encrypt_rotate = true;
+		}
+
 		srv_stats.n_key_requests.inc();
 		key_found = key_version;
 	}
@@ -1380,6 +1391,11 @@ fil_crypt_return_iops(
 	fil_crypt_update_total_stat(state);
 }
 
+bool fil_crypt_must_default_encrypt()
+{
+  return !srv_fil_crypt_rotate_key_age || !srv_encrypt_rotate;
+}
+
 /** Return the next tablespace from default_encrypt_tables.
 @param space   previous tablespace (NULL to start from the start)
 @param recheck whether the removal condition needs to be rechecked after
@@ -1455,7 +1471,7 @@ static fil_space_t *fil_space_next(fil_space_t *space, bool recheck,
   mutex_enter(&fil_system->mutex);
   ut_ad(!space || space->n_pending_ops);
 
-  if (!srv_fil_crypt_rotate_key_age)
+  if (fil_crypt_must_default_encrypt())
     space= fil_system->default_encrypt_next(space, recheck, encrypt);
   else if (!space)
   {
@@ -2448,7 +2464,7 @@ fil_crypt_set_encrypt_tables(
 
 	srv_encrypt_tables = val;
 
-	if (srv_fil_crypt_rotate_key_age == 0) {
+	if (fil_crypt_must_default_encrypt()) {
 		fil_crypt_default_encrypt_tables_fill();
 	}
 
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index 3dc804ea878..a727215b433 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -1464,22 +1464,28 @@ fil_space_create(
 		fil_system->max_assigned_id = id;
 	}
 
+	const bool rotate =
+		(purpose == FIL_TYPE_TABLESPACE
+		 && (mode == FIL_ENCRYPTION_ON
+		     || mode == FIL_ENCRYPTION_OFF || srv_encrypt_tables)
+		 && fil_crypt_must_default_encrypt());
+
 	/* Inform key rotation that there could be something
 	to do */
-	if (purpose == FIL_TYPE_TABLESPACE
-	    && !srv_fil_crypt_rotate_key_age && fil_crypt_threads_event &&
-	    (mode == FIL_ENCRYPTION_ON || mode == FIL_ENCRYPTION_OFF ||
-		    srv_encrypt_tables)) {
+	if (rotate) {
 		/* Key rotation is not enabled, need to inform background
 		encryption threads. */
 		fil_system->default_encrypt_tables.push_back(*space);
 		space->is_in_default_encrypt = true;
 		mutex_exit(&fil_system->mutex);
+	} else {
+		mutex_exit(&fil_system->mutex);
+	}
+
+	if (rotate && srv_n_fil_crypt_threads_started) {
 		mutex_enter(&fil_crypt_threads_mutex);
 		os_event_set(fil_crypt_threads_event);
 		mutex_exit(&fil_crypt_threads_mutex);
-	} else {
-		mutex_exit(&fil_system->mutex);
 	}
 
 	return(space);
diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h
index 3c56315ee9a..af6c930659b 100644
--- a/storage/innobase/include/fil0crypt.h
+++ b/storage/innobase/include/fil0crypt.h
@@ -499,4 +499,10 @@ bool
 fil_space_verify_crypt_checksum(const byte* page, const page_size_t& page_size)
 	MY_ATTRIBUTE((warn_unused_result));
 
+/** Add the tablespace to the rotation list if
+innodb_encrypt_rotate_key_age is 0 or encryption plugin does
+not do key version rotation
+@return whether the tablespace should be added to rotation list */
+bool fil_crypt_must_default_encrypt();
+
 #endif /* fil0crypt_h */

From 7229107e3e2c59febd10b40b243011e8bba94f99 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Tue, 15 Jun 2021 13:13:01 +0530
Subject: [PATCH 13/25] MDEV-25872 InnoDB: Assertion failure in
 row_merge_read_clustered_index upon ALTER on table with indexed virtual
 columns

- InnoDB fails to check DB_COMPUTE_VALUE_FAILED error in
row_merge_read_clustered_index() and wrongly asserts that
the buffer shouldn't be ran out of memory. Alter table
should give warning when the column value is being
truncated.
---
 .../suite/gcol/r/gcol_keys_innodb.result      |  2 ++
 .../suite/gcol/r/innodb_virtual_index.result  | 20 ++++++++++++++
 .../suite/gcol/t/innodb_virtual_index.opt     |  1 +
 .../suite/gcol/t/innodb_virtual_index.test    | 21 +++++++++++++++
 mysql-test/suite/innodb/r/innodb-alter.result |  8 ++++--
 sql/sql_table.cc                              |  8 ++++++
 storage/innobase/row/row0merge.cc             | 27 +++++++++----------
 7 files changed, 71 insertions(+), 16 deletions(-)
 create mode 100644 mysql-test/suite/gcol/t/innodb_virtual_index.opt

diff --git a/mysql-test/suite/gcol/r/gcol_keys_innodb.result b/mysql-test/suite/gcol/r/gcol_keys_innodb.result
index 5a29c64bc20..37b350c0c32 100644
--- a/mysql-test/suite/gcol/r/gcol_keys_innodb.result
+++ b/mysql-test/suite/gcol/r/gcol_keys_innodb.result
@@ -691,6 +691,8 @@ a	b	c
 1	127	0
 SET STATEMENT sql_mode = 'NO_ENGINE_SUBSTITUTION' FOR
 ALTER TABLE t ADD UNIQUE INDEX (c(1));
+Warnings:
+Warning	1264	Out of range value for column 'b' at row 1
 SELECT * FROM t WHERE c = '0';
 a	b	c
 1	127	0
diff --git a/mysql-test/suite/gcol/r/innodb_virtual_index.result b/mysql-test/suite/gcol/r/innodb_virtual_index.result
index 20311a21136..ff339280e8a 100644
--- a/mysql-test/suite/gcol/r/innodb_virtual_index.result
+++ b/mysql-test/suite/gcol/r/innodb_virtual_index.result
@@ -266,3 +266,23 @@ CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 DROP TABLE t1;
+#
+# MDEV-25872 InnoDB: Assertion failure in row_merge_read_clustered_index
+#	upon ALTER on table with indexed virtual columns
+#
+CREATE TABLE t1 (
+id BIGINT AUTO_INCREMENT PRIMARY KEY,
+a INT,
+va INT ZEROFILL AS (a) VIRTUAL,
+b TIMESTAMP,
+c CHAR(204),
+vc CHAR(8),
+KEY(vc,c(64),b,va)
+) ENGINE=InnoDB CHARACTER SET utf32;
+INSERT INTO t1 (id) SELECT NULL FROM seq_1_to_75;
+INSERT IGNORE INTO t1 (id, a) VALUES (NULL, -1);
+Warnings:
+Warning	1264	Out of range value for column 'va' at row 1
+ALTER TABLE t1 FORCE;
+ERROR 22003: Out of range value for column 'va' at row 1
+DROP TABLE t1;
diff --git a/mysql-test/suite/gcol/t/innodb_virtual_index.opt b/mysql-test/suite/gcol/t/innodb_virtual_index.opt
new file mode 100644
index 00000000000..c3f4a891cca
--- /dev/null
+++ b/mysql-test/suite/gcol/t/innodb_virtual_index.opt
@@ -0,0 +1 @@
+--innodb_sort_buffer_size=64k
diff --git a/mysql-test/suite/gcol/t/innodb_virtual_index.test b/mysql-test/suite/gcol/t/innodb_virtual_index.test
index 353841840dc..9e765ca104c 100644
--- a/mysql-test/suite/gcol/t/innodb_virtual_index.test
+++ b/mysql-test/suite/gcol/t/innodb_virtual_index.test
@@ -1,4 +1,5 @@
 --source include/have_innodb.inc
+--source include/have_sequence.inc
 
 # Ensure that the history list length will actually be decremented by purge.
 SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
@@ -281,3 +282,23 @@ ROLLBACK;
 SELECT * FROM t1;
 CHECK TABLE t1;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-25872 InnoDB: Assertion failure in row_merge_read_clustered_index
+--echo #	upon ALTER on table with indexed virtual columns
+--echo #
+
+CREATE TABLE t1 (
+  id BIGINT AUTO_INCREMENT PRIMARY KEY,
+  a INT,
+  va INT ZEROFILL AS (a) VIRTUAL,
+  b TIMESTAMP,
+  c CHAR(204),
+  vc CHAR(8),
+  KEY(vc,c(64),b,va)
+) ENGINE=InnoDB CHARACTER SET utf32;
+INSERT INTO t1 (id) SELECT NULL FROM seq_1_to_75;
+INSERT IGNORE INTO t1 (id, a) VALUES (NULL, -1);
+--error ER_WARN_DATA_OUT_OF_RANGE
+ALTER TABLE t1 FORCE;
+DROP TABLE t1;
diff --git a/mysql-test/suite/innodb/r/innodb-alter.result b/mysql-test/suite/innodb/r/innodb-alter.result
index 80cf14c1725..53e6c95af0f 100644
--- a/mysql-test/suite/innodb/r/innodb-alter.result
+++ b/mysql-test/suite/innodb/r/innodb-alter.result
@@ -1041,7 +1041,9 @@ CREATE TABLE t1 (a INT NOT NULL DEFAULT 0) ENGINE=InnoDB;
 iNSERT INTO t1 VALUES (10);
 ALTER TABLE t1 ADD b DATE NOT NULL DEFAULT if(unix_timestamp()>1,TIMESTAMP'2001-01-01 10:20:30',0);
 affected rows: 0
-info: Records: 0  Duplicates: 0  Warnings: 0
+info: Records: 0  Duplicates: 0  Warnings: 1
+Warnings:
+Note	1265	Data truncated for column 'b' at row 1
 SELECT * FROM t1;
 a	b
 10	2001-01-01
@@ -1050,7 +1052,9 @@ CREATE TABLE t1 (a INT NOT NULL DEFAULT 0) ENGINE=InnoDB;
 iNSERT INTO t1 VALUES (10);
 ALTER TABLE t1 ADD b TIME NOT NULL DEFAULT if(unix_timestamp()>1,TIMESTAMP'2001-01-01 10:20:30',0);
 affected rows: 0
-info: Records: 0  Duplicates: 0  Warnings: 0
+info: Records: 0  Duplicates: 0  Warnings: 1
+Warnings:
+Note	1265	Data truncated for column 'b' at row 1
 SELECT * FROM t1;
 a	b
 10	10:20:30
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 3a3a903bc35..9556cb9f136 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -9680,9 +9680,17 @@ do_continue:;
     if (use_inplace)
     {
       table->s->frm_image= &frm;
+      enum_check_fields save_count_cuted_fields= thd->count_cuted_fields;
+      /*
+        Set the truncated column values of thd as warning for alter table.
+      */
+      thd->count_cuted_fields= CHECK_FIELD_WARN;
       int res= mysql_inplace_alter_table(thd, table_list, table, altered_table,
                                          &ha_alter_info, inplace_supported,
                                          &target_mdl_request, &alter_ctx);
+
+      thd->count_cuted_fields= save_count_cuted_fields;
+
       my_free(const_cast<uchar*>(frm.str));
 
       if (res)
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc
index e3b3f2c2762..93ea4d2d743 100644
--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@@ -524,7 +524,9 @@ row_merge_buf_add(
 	DBUG_ENTER("row_merge_buf_add");
 
 	if (buf->n_tuples >= buf->max_tuples) {
-		DBUG_RETURN(0);
+error:
+		n_row_added = 0;
+		goto end;
 	}
 
 	DBUG_EXECUTE_IF(
@@ -845,11 +847,6 @@ end:
         if (vcol_storage.innobase_record)
 		innobase_free_row_for_vcol(&vcol_storage);
 	DBUG_RETURN(n_row_added);
-
-error:
-        if (vcol_storage.innobase_record)
-		innobase_free_row_for_vcol(&vcol_storage);
-        DBUG_RETURN(0);
 }
 
 /*************************************************************//**
@@ -2567,16 +2564,18 @@ write_buffers:
 						new_table, psort_info, row, ext,
 						&doc_id, conv_heap,
 						&err, &v_heap, eval_table, trx)))) {
-					/* An empty buffer should have enough
-					room for at least one record. */
-					ut_error;
+                                        /* An empty buffer should have enough
+                                        room for at least one record. */
+					ut_ad(err == DB_COMPUTE_VALUE_FAILED
+					      || err == DB_OUT_OF_MEMORY
+					      || err == DB_TOO_BIG_RECORD);
+				} else if (err == DB_SUCCESS) {
+					file->n_rec += rows_added;
+					continue;
 				}
 
-				if (err != DB_SUCCESS) {
-					break;
-				}
-
-				file->n_rec += rows_added;
+				trx->error_key_num = i;
+				break;
 			}
 		}
 

From 7d591cf85090db52478facca4a0c03a2a56aa585 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Tue, 15 Jun 2021 13:14:39 +0530
Subject: [PATCH 14/25] MDEV-24713 Assertion `dict_table_is_comp(index->table)'
 failed in row_merge_buf_add()

- During online alter conversion from compact to redundant,
virtual column field length already set during
innobase_get_computed_value(). Skip the char(n) check for
virtual column in row_merge_buf_add()
---
 .../suite/gcol/r/innodb_virtual_index.result       | 14 ++++++++++++++
 mysql-test/suite/gcol/t/innodb_virtual_index.test  | 12 ++++++++++++
 storage/innobase/row/row0merge.cc                  |  5 ++++-
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/mysql-test/suite/gcol/r/innodb_virtual_index.result b/mysql-test/suite/gcol/r/innodb_virtual_index.result
index ff339280e8a..b8dd161c30e 100644
--- a/mysql-test/suite/gcol/r/innodb_virtual_index.result
+++ b/mysql-test/suite/gcol/r/innodb_virtual_index.result
@@ -286,3 +286,17 @@ Warning	1264	Out of range value for column 'va' at row 1
 ALTER TABLE t1 FORCE;
 ERROR 22003: Out of range value for column 'va' at row 1
 DROP TABLE t1;
+#
+# MDEV-24713 Assertion `dict_table_is_comp(index->table)' failed
+#		in row_merge_buf_add()
+#
+CREATE TABLE t1 (id INT PRIMARY KEY, a CHAR(3),
+b CHAR(8) AS (a) VIRTUAL, KEY(b))
+ROW_FORMAT=REDUNDANT ENGINE=InnoDB
+CHARACTER SET utf8;
+INSERT INTO t1 (id,a) VALUES (1,'foo');
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	note	Table does not support optimize, doing recreate + analyze instead
+test.t1	optimize	status	OK
+DROP TABLE t1;
diff --git a/mysql-test/suite/gcol/t/innodb_virtual_index.test b/mysql-test/suite/gcol/t/innodb_virtual_index.test
index 9e765ca104c..94c3b7f9204 100644
--- a/mysql-test/suite/gcol/t/innodb_virtual_index.test
+++ b/mysql-test/suite/gcol/t/innodb_virtual_index.test
@@ -302,3 +302,15 @@ INSERT IGNORE INTO t1 (id, a) VALUES (NULL, -1);
 --error ER_WARN_DATA_OUT_OF_RANGE
 ALTER TABLE t1 FORCE;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-24713 Assertion `dict_table_is_comp(index->table)' failed
+--echo #		in row_merge_buf_add()
+--echo #
+CREATE TABLE t1 (id INT PRIMARY KEY, a CHAR(3),
+		 b CHAR(8) AS (a) VIRTUAL, KEY(b))
+		 ROW_FORMAT=REDUNDANT ENGINE=InnoDB
+		 CHARACTER SET utf8;
+INSERT INTO t1 (id,a) VALUES (1,'foo');
+OPTIMIZE TABLE t1;
+DROP TABLE t1;
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc
index 93ea4d2d743..9e6179585b1 100644
--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@@ -691,7 +691,10 @@ error:
 				continue;
 			}
 
-			if (field->len != UNIV_SQL_NULL
+			/* innobase_get_computed_value() sets the
+			length of the virtual column field. */
+			if (v_col == NULL
+			    && field->len != UNIV_SQL_NULL
 			    && col->mtype == DATA_MYSQL
 			    && col->len != field->len) {
 				if (conv_heap != NULL) {

From 1c35a3f6fd92704d7a135a81c7752f5058aaede5 Mon Sep 17 00:00:00 2001
From: Eugene Kosov <claprix@yandex.ru>
Date: Tue, 15 Jun 2021 13:09:15 +0300
Subject: [PATCH 15/25] fix clang build

a new warning -Wunused-but-set-variable was introduced recently to clang
---
 mysys/mf_iocache.c                 |  3 +--
 sql/item_func.cc                   |  2 --
 sql/sql_select.cc                  | 10 +---------
 storage/innobase/fil/fil0crypt.cc  |  2 --
 storage/innobase/fsp/fsp0fsp.cc    |  2 --
 storage/innobase/row/row0ftsort.cc |  2 --
 storage/maria/ma_check.c           |  8 ++++++--
 storage/myisam/mi_check.c          |  8 ++++++--
 8 files changed, 14 insertions(+), 23 deletions(-)

diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c
index 8ade76e922e..0b19aaf9015 100644
--- a/mysys/mf_iocache.c
+++ b/mysys/mf_iocache.c
@@ -1384,7 +1384,7 @@ static void copy_to_read_buffer(IO_CACHE *write_cache,
 
 static int _my_b_seq_read(IO_CACHE *info, uchar *Buffer, size_t Count)
 {
-  size_t length, diff_length, left_length= 0, save_count, max_length;
+  size_t length, diff_length, save_count, max_length;
   my_off_t pos_in_file;
   save_count=Count;
 
@@ -1435,7 +1435,6 @@ static int _my_b_seq_read(IO_CACHE *info, uchar *Buffer, size_t Count)
       */
       goto read_append_buffer;
     }
-    left_length+=length;
     diff_length=0;
   }
 
diff --git a/sql/item_func.cc b/sql/item_func.cc
index 7b39b7710fb..6b8025c9d0f 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -2756,7 +2756,6 @@ bool Item_func_min_max::fix_length_and_dec()
   max_length=0;
   maybe_null=0;
   Item_result tmp_cmp_type= args[0]->cmp_type();
-  uint string_type_count= 0;
   uint temporal_type_count= 0;
   enum_field_types temporal_field_type= MYSQL_TYPE_DATETIME;
 
@@ -2769,7 +2768,6 @@ bool Item_func_min_max::fix_length_and_dec()
     if (args[i]->maybe_null)
       maybe_null= 1;
     tmp_cmp_type= item_cmp_type(tmp_cmp_type, args[i]->cmp_type());
-    string_type_count+= args[i]->cmp_type() == STRING_RESULT;
     if (args[i]->cmp_type() == TIME_RESULT)
     {
       if (!temporal_type_count)
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index ce706209017..fcdb671941d 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -3928,7 +3928,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
   int error= 0;
   TABLE *UNINIT_VAR(table); /* inited in all loops */
   uint i,table_count,const_count,key;
-  table_map found_const_table_map, all_table_map, found_ref, refs;
+  table_map found_const_table_map, all_table_map;
   key_map const_ref, eq_part;
   bool has_expensive_keyparts;
   TABLE **table_vector;
@@ -4191,7 +4191,6 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
   {
     ref_changed = 0;
   more_const_tables_found:
-    found_ref=0;
 
     /*
       We only have to loop from stat_vector + const_count as
@@ -4280,7 +4279,6 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
 	  key=keyuse->key;
 	  s->keys.set_bit(key);               // TODO: remove this ?
 
-	  refs=0;
           const_ref.clear_all();
 	  eq_part.clear_all();
           has_expensive_keyparts= false;
@@ -4296,8 +4294,6 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
                 if (keyuse->val->is_expensive())
                   has_expensive_keyparts= true;
               }
-	      else
-		refs|=keyuse->used_tables;
 	      eq_part.set_bit(keyuse->keypart);
 	    }
 	    keyuse++;
@@ -4348,8 +4344,6 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
 		  found_const_table_map|= table->map;
 	        break;
 	      }
-	      else
-	        found_ref|= refs;      // Table is const if all refs are const
 	    }
             else if (base_const_ref == base_eq_part)
               s->const_keys.set_bit(key);
@@ -26190,7 +26184,6 @@ static bool get_range_limit_read_cost(const JOIN_TAB *tab,
     */
     if (tab)
     {
-      key_part_map const_parts= 0;
       key_part_map map= 1;
       uint kp;
       /* Find how many key parts would be used by ref(const) */
@@ -26198,7 +26191,6 @@ static bool get_range_limit_read_cost(const JOIN_TAB *tab,
       {
         if (!(table->const_key_parts[keynr] & map))
           break;
-        const_parts |= map;
       }
       
       if (kp > 0)
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 8195716f722..94899ed0f65 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -1012,13 +1012,11 @@ static bool fil_crypt_start_encrypting_space(fil_space_t* space)
 		/* 4 - sync tablespace before publishing crypt data */
 
 		bool success = false;
-		ulint sum_pages = 0;
 
 		do {
 			ulint n_pages = 0;
 			success = buf_flush_lists(ULINT_MAX, end_lsn, &n_pages);
 			buf_flush_wait_batch_end(NULL, BUF_FLUSH_LIST);
-			sum_pages += n_pages;
 		} while (!success);
 
 		/* 5 - publish crypt data */
diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc
index 58aa18ac323..cb5eff07f88 100644
--- a/storage/innobase/fsp/fsp0fsp.cc
+++ b/storage/innobase/fsp/fsp0fsp.cc
@@ -2719,7 +2719,6 @@ fsp_reserve_free_extents(
 	ulint		n_free;
 	ulint		n_free_up;
 	ulint		reserve;
-	size_t		total_reserved = 0;
 
 	ut_ad(mtr);
 	*n_reserved = n_ext;
@@ -2801,7 +2800,6 @@ try_again:
 	}
 try_to_extend:
 	if (ulint n = fsp_try_extend_data_file(space, space_header, mtr)) {
-		total_reserved += n;
 		goto try_again;
 	}
 
diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc
index 2ca930e0934..29e126290cf 100644
--- a/storage/innobase/row/row0ftsort.cc
+++ b/storage/innobase/row/row0ftsort.cc
@@ -770,7 +770,6 @@ fts_parallel_tokenization(
 	row_merge_block_t**	crypt_block;
 	int			tmpfd[FTS_NUM_AUX_INDEX];
 	ulint			mycount[FTS_NUM_AUX_INDEX];
-	ib_uint64_t		total_rec = 0;
 	ulint			num_doc_processed = 0;
 	doc_id_t		last_doc_id = 0;
 	mem_heap_t*		blob_heap = NULL;
@@ -1038,7 +1037,6 @@ exit:
 			goto func_exit;
 		}
 
-		total_rec += merge_file[i]->n_rec;
 		close(tmpfd[i]);
 	}
 
diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c
index 2795528a044..cb4585d0f62 100644
--- a/storage/maria/ma_check.c
+++ b/storage/maria/ma_check.c
@@ -4191,7 +4191,7 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info,
 			const char * name, my_bool rep_quick)
 {
   int got_error;
-  uint i,key, total_key_length, istep;
+  uint i,key, istep;
   ha_rows start_records;
   my_off_t new_header_length,del;
   File new_file;
@@ -4353,7 +4353,9 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info,
     _ma_check_print_error(param,"Not enough memory for key!");
     goto err;
   }
-  total_key_length=0;
+#ifdef USING_SECOND_APPROACH
+  uint total_key_length=0;
+#endif
   rec_per_key_part= param->new_rec_per_key_part;
   share->state.state.records=share->state.state.del=share->state.split=0;
   share->state.state.empty=0;
@@ -4422,7 +4424,9 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info,
       if (keyseg->flag & HA_NULL_PART)
         sort_param[i].key_length++;
     }
+#ifdef USING_SECOND_APPROACH
     total_key_length+=sort_param[i].key_length;
+#endif
 
     if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
     {
diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c
index d9b9bb5af4a..8bae409d1e6 100644
--- a/storage/myisam/mi_check.c
+++ b/storage/myisam/mi_check.c
@@ -2614,7 +2614,7 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info,
 			const char * name, int rep_quick)
 {
   int got_error;
-  uint i,key, total_key_length, istep;
+  uint i,key, istep;
   ulong rec_length;
   ha_rows start_records;
   my_off_t new_header_length,del;
@@ -2800,7 +2800,9 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info,
     mi_check_print_error(param,"Not enough memory for key!");
     goto err;
   }
-  total_key_length=0;
+#ifdef USING_SECOND_APPROACH
+  uint total_key_length=0;
+#endif
   rec_per_key_part= param->rec_per_key_part;
   info->state->records=info->state->del=share->state.split=0;
   info->state->empty=0;
@@ -2869,7 +2871,9 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info,
       if (keyseg->flag & HA_NULL_PART)
         sort_param[i].key_length++;
     }
+#ifdef USING_SECOND_APPROACH
     total_key_length+=sort_param[i].key_length;
+#endif
 
     if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
     {

From 18d5be5b54b1a05e6107a1c5828d9eed9cf18636 Mon Sep 17 00:00:00 2001
From: Julius Goryavsky <julius.goryavsky@mariadb.com>
Date: Wed, 9 Jun 2021 03:41:37 +0200
Subject: [PATCH 16/25] MDEV-25880: rsync may be mistakenly killed when
 overlapping SST

This commit fixes a bug was originally discovered during the
galera_nbo_sst_slave mtr test for 10.6 branch. However it is
relevant for all versions and can lead to intermittent SST
crashes via rsync on very fast server restarts - when a new
SST process (for example, after starting a new server instance)
overlaps the old SST process started by the previous, already
terminated server. This overlap can result in the new rsync
being killed instead of the old rsync, or the pid file from
the new rsync being killed, which then lead to problems.
---
 scripts/wsrep_sst_common.sh |  2 +-
 scripts/wsrep_sst_rsync.sh  | 28 ++++++++++++++++++++++++----
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh
index 05944ef6035..c2f31b2818d 100644
--- a/scripts/wsrep_sst_common.sh
+++ b/scripts/wsrep_sst_common.sh
@@ -1190,7 +1190,6 @@ trim_string()
 check_pid()
 {
     local pid_file="$1"
-    local remove=${2:-0}
     if [ -r "$pid_file" ]; then
         local pid=$(cat "$pid_file" 2>/dev/null)
         if [ -n "$pid" ]; then
@@ -1201,6 +1200,7 @@ check_pid()
                 fi
             fi
         fi
+        local remove=${2:-0}
         if [ $remove -eq 1 ]; then
             rm -f "$pid_file"
         fi
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index 19a4d19fded..a602af79af0 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -68,6 +68,8 @@ cleanup_joiner()
     if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
         wsrep_cleanup_progress_file
     fi
+
+    [ -f "$SST_PID" ] && rm -f "$SST_PID"
 }
 
 check_pid_and_port()
@@ -281,6 +283,7 @@ then
     *)
         wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'"
         exit 22 # EINVAL
+        ;;
     esac
     if [ -z "$CAFILE_OPT" ]; then
         wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file"
@@ -499,6 +502,21 @@ elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
 then
     check_sockets_utils
 
+    SST_PID="$WSREP_SST_OPT_DATA/wsrep_rsync_sst.pid"
+
+    # give some time for lingering stunnel from previous SST to complete
+    check_round=0
+    while check_pid "$SST_PID" 0
+    do
+        wsrep_log_info "previous SST not completed, waiting for it to exit"
+        check_round=$(( check_round + 1 ))
+        if [ $check_round -eq 10 ]; then
+            wsrep_log_error "SST script already running."
+            exit 114 # EALREADY
+        fi
+        sleep 1
+    done
+
     # give some time for lingering stunnel from previous SST to complete
     check_round=0
     while check_pid "$STUNNEL_PID" 1
@@ -583,12 +601,14 @@ EOF
         RSYNC_ADDR="*"
     fi
 
+    echo $$ > "$SST_PID"
+
     if [ -z "$STUNNEL" ]
     then
         rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS &
         RSYNC_REAL_PID=$!
-        TRANSFER_REAL_PID="$RSYNC_REAL_PID"
-        TRANSFER_PID=$RSYNC_PID
+        TRANSFER_REAL_PID=$RSYNC_REAL_PID
+        TRANSFER_PID="$RSYNC_PID"
     else
         # Let's check if the path to the config file contains a space?
         if [ "${RSYNC_CONF#* }" = "$RSYNC_CONF" ]; then
@@ -631,8 +651,8 @@ EOF
         fi
         stunnel "$STUNNEL_CONF" &
         STUNNEL_REAL_PID=$!
-        TRANSFER_REAL_PID="$STUNNEL_REAL_PID"
-        TRANSFER_PID=$STUNNEL_PID
+        TRANSFER_REAL_PID=$STUNNEL_REAL_PID
+        TRANSFER_PID="$STUNNEL_PID"
     fi
 
     if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]

From 2edb8e12e10179b970007b3e1d5c465b9d0e110e Mon Sep 17 00:00:00 2001
From: Julius Goryavsky <julius.goryavsky@mariadb.com>
Date: Tue, 15 Jun 2021 06:24:38 +0200
Subject: [PATCH 17/25] MDEV-25880 part 2: Improving reliability of the SST
 scripts

Additional improvements aimed at improving operational
reliability of the SST scripts:

1) Script need to give rsync and stunnel a short time to
   terminate after "kill -9" before the first PID check
   using ps utility;
2) The temporary file used to create the binlog index could
   sometimes remain in the data directory if tar failed and
   then may be reused without being cleaned up (the next
   time when SST was run) - now it's fixed;
3) The temporary file used to build the binlog index is now
   created using mktemp and, if this variable is present in
   the configuration file, in tmpdir;
4) Checking the secret tag in SST via rsync is made faster
   and does not require creating a temporary file, which
   could remain in the data directory in case of failure;
5) Added "-F" option to grep to check the tag when using
   mariabackup/xtrabackup-v2 - to avoid possible collisions
   in case of special characters in the tag value (unlikely
   scenario, but the new check is more reliable).
---
 scripts/wsrep_sst_common.sh        |  7 ++--
 scripts/wsrep_sst_mariabackup.sh   |  6 +--
 scripts/wsrep_sst_rsync.sh         | 64 ++++++++++++++++++------------
 scripts/wsrep_sst_xtrabackup-v2.sh |  6 +--
 4 files changed, 49 insertions(+), 34 deletions(-)

diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh
index c2f31b2818d..4dedecb439f 100644
--- a/scripts/wsrep_sst_common.sh
+++ b/scripts/wsrep_sst_common.sh
@@ -1223,7 +1223,7 @@ check_pid()
 #
 cleanup_pid()
 {
-    local pid="$1"
+    local pid=$1
     local pid_file="${2:-}"
     local config="${3:-}"
 
@@ -1241,8 +1241,9 @@ cleanup_pid()
                            round=8
                            force=1
                            kill -9 $pid >/dev/null 2>&1
+                           sleep 0.5
                        else
-                           return 1;
+                           return 1
                        fi
                    fi
                 done
@@ -1254,7 +1255,7 @@ cleanup_pid()
     fi
 
     [ -n "$pid_file" ] && [ -f "$pid_file" ] && rm -f "$pid_file"
-    [ -n "$config" ]   && [ -f "$config"   ] && rm -f "$config"
+    [ -n "$config" ]   && [ -f "$config" ]   && rm -f "$config"
 
     return 0
 }
diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index 7f97d9e8dea..339a8fcf4a5 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -741,15 +741,15 @@ recv_joiner()
         fi
 
         # check donor supplied secret
-        SECRET=$(grep -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
+        SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
         if [ "$SECRET" != "$MY_SECRET" ]; then
             wsrep_log_error "Donor does not know my secret!"
             wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'"
             exit 32
         fi
 
-        # remove secret from magic file
-        grep -v -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
+        # remove secret from the magic file
+        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
         mv "$MAGIC_FILE.new" "$MAGIC_FILE"
     fi
 }
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index a602af79af0..fc9f5017937 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -429,7 +429,7 @@ EOF
             exit 255 # unknown error
         fi
 
-        # second, we transfer InnoDB log files
+        # second, we transfer InnoDB and Aria log files
         rsync ${STUNNEL:+--rsh="$STUNNEL"} \
               --owner --group --perms --links --specials \
               --ignore-times --inplace --dirs --delete --quiet \
@@ -504,20 +504,20 @@ then
 
     SST_PID="$WSREP_SST_OPT_DATA/wsrep_rsync_sst.pid"
 
-    # give some time for lingering stunnel from previous SST to complete
+    # give some time for previous SST to complete:
     check_round=0
     while check_pid "$SST_PID" 0
     do
-        wsrep_log_info "previous SST not completed, waiting for it to exit"
+        wsrep_log_info "previous SST is not completed, waiting for it to exit"
         check_round=$(( check_round + 1 ))
         if [ $check_round -eq 10 ]; then
-            wsrep_log_error "SST script already running."
+            wsrep_log_error "previous SST script still running."
             exit 114 # EALREADY
         fi
         sleep 1
     done
 
-    # give some time for lingering stunnel from previous SST to complete
+    # give some time for stunnel from the previous SST to complete:
     check_round=0
     while check_pid "$STUNNEL_PID" 1
     do
@@ -534,7 +534,7 @@ then
     RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
     RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
 
-    # give some time for lingering rsync from previous SST to complete
+    # give some time for rsync from the previous SST to complete:
     check_round=0
     while check_pid "$RSYNC_PID" 1
     do
@@ -711,35 +711,49 @@ EOF
 
             # Clean up old binlog files first
             rm -f "$BINLOG_FILENAME".[0-9]*
-            [ -f "$binlog_index" ] && rm "$binlog_index"
+            [ -f "$binlog_index" ] && rm -f "$binlog_index"
+
+            # Create a temporary file:
+            tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir')
+            if [ -z "$tmpdir" ]; then
+               tmpfile="$(mktemp)"
+            else
+               tmpfile=$(mktemp "--tmpdir=$tmpdir")
+            fi
 
             wsrep_log_info "Extracting binlog files:"
-            tar -xvf "$BINLOG_TAR_FILE" >> _binlog_tmp_files_$!
+            if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then
+                wsrep_log_error "Error unpacking tar file with binlog files"
+                rm -f "$tmpfile"
+                exit 32
+            fi
+
+            # Rebuild binlog index:
             while read bin_file; do
                 echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index"
-            done < _binlog_tmp_files_$!
-            rm -f _binlog_tmp_files_$!
+            done < "$tmpfile"
+            rm -f "$tmpfile"
 
             cd "$OLD_PWD"
         fi
     fi
 
-    if [ -r "$MAGIC_FILE" ]
-    then
-        # check donor supplied secret
-        SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
-        if [ "$SECRET" != "$MY_SECRET" ]; then
-            wsrep_log_error "Donor does not know my secret!"
-            wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'"
-            exit 32
+    if [ -r "$MAGIC_FILE" ]; then
+        if [ -n "$MY_SECRET" ]; then
+            # check donor supplied secret
+            SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
+            if [ "$SECRET" != "$MY_SECRET" ]; then
+                wsrep_log_error "Donor does not know my secret!"
+                wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'"
+                exit 32
+            fi
+            # remove secret from the magic file, and output
+            # the UUID:seqno & wsrep_gtid_domain_id:
+            grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE"
+        else
+            # Output the UUID:seqno and wsrep_gtid_domain_id:
+            cat "$MAGIC_FILE"
         fi
-
-        # remove secret from magic file
-        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
-
-        mv "$MAGIC_FILE.new" "$MAGIC_FILE"
-        # UUID:seqno & wsrep_gtid_domain_id is received here.
-        cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id
     else
         # this message should cause joiner to abort
         echo "rsync process ended without creating '$MAGIC_FILE'"
diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh
index 24bff12219d..d76dc346a82 100644
--- a/scripts/wsrep_sst_xtrabackup-v2.sh
+++ b/scripts/wsrep_sst_xtrabackup-v2.sh
@@ -750,15 +750,15 @@ recv_joiner()
         fi
 
         # check donor supplied secret
-        SECRET=$(grep -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
+        SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
         if [ "$SECRET" != "$MY_SECRET" ]; then
             wsrep_log_error "Donor does not know my secret!"
             wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'"
             exit 32
         fi
 
-        # remove secret from magic file
-        grep -v -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
+        # remove secret from the magic file
+        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
         mv "$MAGIC_FILE.new" "$MAGIC_FILE"
     fi
 }

From c307dc6efde682c0768dc900818f4c0b418f9c6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 16 Jun 2021 07:50:04 +0300
Subject: [PATCH 18/25] Remove a unused variable

In commit 1c35a3f6fd92704d7a135a81c7752f5058aaede5 a useless
computation that used the variable was removed.
---
 storage/innobase/fsp/fsp0fsp.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc
index cb5eff07f88..43989d57db8 100644
--- a/storage/innobase/fsp/fsp0fsp.cc
+++ b/storage/innobase/fsp/fsp0fsp.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2020, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2799,7 +2799,7 @@ try_again:
 		return(true);
 	}
 try_to_extend:
-	if (ulint n = fsp_try_extend_data_file(space, space_header, mtr)) {
+	if (fsp_try_extend_data_file(space, space_header, mtr)) {
 		goto try_again;
 	}
 

From 35b57c37bbbfa116da4a454df5892984550b151a Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Thu, 13 May 2021 17:54:15 +0200
Subject: [PATCH 19/25] MDEV-25617 10.5.10 upgrade: "scriptlet / line 6 : [:
 is-active : binary operator expected"

---
 support-files/rpm/server-posttrans.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/support-files/rpm/server-posttrans.sh b/support-files/rpm/server-posttrans.sh
index 1525b592735..3250423da8f 100644
--- a/support-files/rpm/server-posttrans.sh
+++ b/support-files/rpm/server-posttrans.sh
@@ -3,7 +3,7 @@ if [ -r %{restart_flag} ] ; then
   # only restart the server if it was alredy running
   if [ -x /usr/bin/systemctl ] ; then
     /usr/bin/systemctl daemon-reload > /dev/null 2>&1
-    if [ /usr/bin/systemctl is-active mysql ]; then
+    if /usr/bin/systemctl is-active mysql; then
       /usr/bin/systemctl restart mysql > /dev/null 2>&1
     else
       /usr/bin/systemctl try-restart mariadb.service > /dev/null 2>&1

From cd1a195b22d2dac248228b6b7ecbd0a54997ecf7 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Thu, 17 Jun 2021 16:10:11 +0530
Subject: [PATCH 20/25] MDEV-25947 innodb_fts.misc_debug fails in buildbot

- Make innodb_fts.misc_debug test case more stable.
---
 mysql-test/suite/innodb_fts/r/misc_debug.result | 10 ++++------
 mysql-test/suite/innodb_fts/t/misc_debug.test   | 12 ++++--------
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/mysql-test/suite/innodb_fts/r/misc_debug.result b/mysql-test/suite/innodb_fts/r/misc_debug.result
index b162b2f7415..70b239bb189 100644
--- a/mysql-test/suite/innodb_fts/r/misc_debug.result
+++ b/mysql-test/suite/innodb_fts/r/misc_debug.result
@@ -31,19 +31,17 @@ DROP TABLE t2, t1;
 #
 CREATE TABLE t1(a INT, b TEXT, c TEXT, FULLTEXT INDEX(b)) ENGINE=InnoDB;
 connect con1,localhost,root,,test;
-SET DEBUG_SYNC='innodb_inplace_alter_table_enter SIGNAL s1 WAIT_FOR g1';
+SET DEBUG_DBUG="+d,innodb_OOM_inplace_alter";
 SET DEBUG_SYNC='innodb_commit_inplace_alter_table_enter SIGNAL s2 WAIT_FOR g2';
 ALTER TABLE t1 ADD FULLTEXT(c);
 connection default;
-SET DEBUG_SYNC='now WAIT_FOR s1';
-KILL QUERY @id;
-SET DEBUG_SYNC='now SIGNAL g1 WAIT_FOR s2';
+SET DEBUG_SYNC='now WAIT_FOR s2';
 START TRANSACTION;
 SELECT * FROM t1;
 a	b	c
-SET DEBUG_SYNC='now SIGNAL s2';
+SET DEBUG_SYNC='now SIGNAL g2';
 connection con1;
-ERROR 70100: Query execution was interrupted
+ERROR HY000: Out of memory; check if mysqld or some other process uses all available memory; if not, you may have to use 'ulimit' to allow mysqld to use more memory or you can add more swap space
 disconnect con1;
 connection default;
 SET DEBUG_SYNC=RESET;
diff --git a/mysql-test/suite/innodb_fts/t/misc_debug.test b/mysql-test/suite/innodb_fts/t/misc_debug.test
index 461e3f1d9d4..e42fc09bcd5 100644
--- a/mysql-test/suite/innodb_fts/t/misc_debug.test
+++ b/mysql-test/suite/innodb_fts/t/misc_debug.test
@@ -60,20 +60,16 @@ DROP TABLE t2, t1;
 --echo #
 CREATE TABLE t1(a INT, b TEXT, c TEXT, FULLTEXT INDEX(b)) ENGINE=InnoDB;
 connect(con1,localhost,root,,test);
-let $ID= `SELECT @id := CONNECTION_ID()`;
-SET DEBUG_SYNC='innodb_inplace_alter_table_enter SIGNAL s1 WAIT_FOR g1';
+SET DEBUG_DBUG="+d,innodb_OOM_inplace_alter";
 SET DEBUG_SYNC='innodb_commit_inplace_alter_table_enter SIGNAL s2 WAIT_FOR g2';
 send ALTER TABLE t1 ADD FULLTEXT(c);
 connection default;
-SET DEBUG_SYNC='now WAIT_FOR s1';
-let $ignore= `SELECT @id := $ID`;
-KILL QUERY @id;
-SET DEBUG_SYNC='now SIGNAL g1 WAIT_FOR s2';
+SET DEBUG_SYNC='now WAIT_FOR s2';
 START TRANSACTION;
 SELECT * FROM t1;
-SET DEBUG_SYNC='now SIGNAL s2';
+SET DEBUG_SYNC='now SIGNAL g2';
 connection con1;
---error ER_QUERY_INTERRUPTED
+--error ER_OUT_OF_RESOURCES
 reap;
 disconnect con1;
 connection default;

From a5f6eca50db3b8c6abddc1e373441827c4655edc Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Sat, 19 Jun 2021 00:22:15 +0200
Subject: [PATCH 21/25] spider tests aren't big

and *never* disable tests in suite.pm based on $::opt_big_test,
this will make the test skipped both as too big (for ./mtr)
and as too small (for ./mtr --big --big).
---
 storage/spider/mysql-test/spider/bg/suite.pm      | 1 -
 storage/spider/mysql-test/spider/bugfix/suite.pm  | 1 -
 storage/spider/mysql-test/spider/handler/suite.pm | 1 -
 storage/spider/mysql-test/spider/suite.pm         | 1 -
 4 files changed, 4 deletions(-)

diff --git a/storage/spider/mysql-test/spider/bg/suite.pm b/storage/spider/mysql-test/spider/bg/suite.pm
index f106147deb6..171fa6c4f01 100644
--- a/storage/spider/mysql-test/spider/bg/suite.pm
+++ b/storage/spider/mysql-test/spider/bg/suite.pm
@@ -4,7 +4,6 @@ package My::Suite::Spider;
 
 return "No Spider engine" unless $ENV{HA_SPIDER_SO};
 return "Not run for embedded server" if $::opt_embedded_server;
-return "Test needs --big-test" unless $::opt_big_test;
 
 sub is_default { 1 }
 
diff --git a/storage/spider/mysql-test/spider/bugfix/suite.pm b/storage/spider/mysql-test/spider/bugfix/suite.pm
index f106147deb6..171fa6c4f01 100644
--- a/storage/spider/mysql-test/spider/bugfix/suite.pm
+++ b/storage/spider/mysql-test/spider/bugfix/suite.pm
@@ -4,7 +4,6 @@ package My::Suite::Spider;
 
 return "No Spider engine" unless $ENV{HA_SPIDER_SO};
 return "Not run for embedded server" if $::opt_embedded_server;
-return "Test needs --big-test" unless $::opt_big_test;
 
 sub is_default { 1 }
 
diff --git a/storage/spider/mysql-test/spider/handler/suite.pm b/storage/spider/mysql-test/spider/handler/suite.pm
index b023e5206ef..af267d047a4 100644
--- a/storage/spider/mysql-test/spider/handler/suite.pm
+++ b/storage/spider/mysql-test/spider/handler/suite.pm
@@ -4,7 +4,6 @@ package My::Suite::Spider;
 
 return "No Spider engine" unless $ENV{HA_SPIDER_SO};
 return "Not run for embedded server" if $::opt_embedded_server;
-return "Test needs --big-test" unless $::opt_big_test;
 
 sub is_default { 1 }
 
diff --git a/storage/spider/mysql-test/spider/suite.pm b/storage/spider/mysql-test/spider/suite.pm
index f106147deb6..171fa6c4f01 100644
--- a/storage/spider/mysql-test/spider/suite.pm
+++ b/storage/spider/mysql-test/spider/suite.pm
@@ -4,7 +4,6 @@ package My::Suite::Spider;
 
 return "No Spider engine" unless $ENV{HA_SPIDER_SO};
 return "Not run for embedded server" if $::opt_embedded_server;
-return "Test needs --big-test" unless $::opt_big_test;
 
 sub is_default { 1 }
 

From 068246c006b8f691982194529a32bff7eb010694 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Sat, 19 Jun 2021 00:45:49 +0200
Subject: [PATCH 22/25] fix spider tests for --ps

spider tests use

--let $var= many;sql;statements
--eval $var

and this doesn't work in ps
---
 storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test     | 4 ++++
 storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test     | 4 ++++
 .../spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test    | 2 ++
 storage/spider/mysql-test/spider/t/partition_mrr.test         | 1 +
 4 files changed, 11 insertions(+)

diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test
index b72facd11a6..3f522b4f6f5 100644
--- a/storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test
+++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test
@@ -22,7 +22,9 @@ USE auto_test_remote;
 --connection child2_1
 --disable_query_log
 echo CHILD2_1_CREATE_TABLES;
+--disable_ps_protocol
 eval $CHILD2_1_CREATE_TABLES;
+--enable_ps_protocol
 --enable_query_log
 TRUNCATE TABLE mysql.general_log;
 
@@ -66,7 +68,9 @@ SELECT a, b, c FROM tbl_a PARTITION (pt2,pt3);
 
 --connection child2_1
 eval $CHILD2_1_SELECT_ARGUMENT1;
+--disable_ps_protocol
 eval $CHILD2_1_SELECT_TABLES;
+--enable_ps_protocol
 
 --echo
 --echo deinit
diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test
index be4aa6a6330..16836cfce28 100644
--- a/storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test
+++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test
@@ -22,7 +22,9 @@ USE auto_test_remote;
 --connection child2_1
 --disable_query_log
 echo CHILD2_1_CREATE_TABLES;
+--disable_ps_protocol
 eval $CHILD2_1_CREATE_TABLES;
+--enable_ps_protocol
 --enable_query_log
 TRUNCATE TABLE mysql.general_log;
 
@@ -78,7 +80,9 @@ SELECT STRAIGHT_JOIN b.a, b.b FROM tb_l a, tbl_a b WHERE a.a = b.a;
 --connection child2_1
 SET NAMES utf8;
 eval $CHILD2_1_SELECT_ARGUMENT1;
+--disable_ps_protocol
 eval $CHILD2_1_SELECT_TABLES;
+--enable_ps_protocol
 
 --echo
 --echo deinit
diff --git a/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test b/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test
index 0102155b5ab..4f980140a31 100644
--- a/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test
+++ b/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test
@@ -49,6 +49,8 @@ TRUNCATE TABLE mysql.general_log;
 SELECT * FROM tbl_a ORDER BY pkey;
 
 --connection child2_1
+# in --ps a query is logged differently in a general log
+replace_regex /order by t0.`pkey`/order by `pkey`/;
 eval $CHILD2_1_SELECT_ARGUMENT1;
 eval $CHILD2_1_SELECT_TABLES;
 
diff --git a/storage/spider/mysql-test/spider/t/partition_mrr.test b/storage/spider/mysql-test/spider/t/partition_mrr.test
index e7fedce33e6..2816d65cadb 100644
--- a/storage/spider/mysql-test/spider/t/partition_mrr.test
+++ b/storage/spider/mysql-test/spider/t/partition_mrr.test
@@ -1,3 +1,4 @@
+--source include/no_protocol.inc
 --source ../include/partition_mrr_init.inc
 if (!$HAVE_PARTITION)
 {

From 773a07b65517327add6348c045cee14bdf489fe0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 21 Jun 2021 08:18:34 +0300
Subject: [PATCH 23/25] Remove Travis CI status

Builds on travis-ci.org ceased on 2021-06-15.
---
 README.md | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/README.md b/README.md
index 03e71af7652..d208d171377 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,6 @@
 Code status:
 ------------
 
-* [![Travis CI status](https://secure.travis-ci.org/MariaDB/server.png?branch=10.2)](https://travis-ci.org/MariaDB/server) travis-ci.org (10.2 branch)
 * [![Appveyor CI status](https://ci.appveyor.com/api/projects/status/4u6pexmtpuf8jq66?svg=true)](https://ci.appveyor.com/project/rasmushoj/server) ci.appveyor.com
 
 ## MariaDB: drop-in replacement for MySQL
@@ -76,10 +75,3 @@ https://bugs.mysql.com
 
 The code for MariaDB, including all revision history, can be found at:
 https://github.com/MariaDB/server
-
-***************************************************************************
-
-Code status:
-------------
-
-* [![tests status](https://secure.travis-ci.org/MariaDB/server.png?branch=10.2)](https://travis-ci.org/MariaDB/server) travis-ci.org (10.2 branch)

From 241d30d3faac0fbbcb7bcfad3dcc118bb03eceb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 21 Jun 2021 12:09:00 +0300
Subject: [PATCH 24/25] After-merge fixes for MDEV-14180

---
 .../suite/encryption/r/innodb-first-page-read.result      | 2 ++
 .../suite/encryption/r/innodb-key-rotation-disable.result | 2 ++
 mysql-test/suite/encryption/t/innodb-first-page-read.test | 8 +++++++-
 storage/innobase/fil/fil0fil.cc                           | 8 +++++---
 4 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/mysql-test/suite/encryption/r/innodb-first-page-read.result b/mysql-test/suite/encryption/r/innodb-first-page-read.result
index 29253885e83..4c2a2116669 100644
--- a/mysql-test/suite/encryption/r/innodb-first-page-read.result
+++ b/mysql-test/suite/encryption/r/innodb-first-page-read.result
@@ -66,6 +66,8 @@ VARIABLE_VALUE <= 29
 1
 set global innodb_encrypt_tables=OFF;
 # wait until tables are decrypted
+SET GLOBAL innodb_max_dirty_pages_pct=0.0;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm=0.0;
 # result should be actual number of tables except remote tables could be read twice
 # i.e. < 23 + 3*2 = 29
 SELECT VARIABLE_VALUE <= 29  FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'innodb_pages0_read';
diff --git a/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result b/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result
index 7d0267d5057..f27f5517e09 100644
--- a/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result
+++ b/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result
@@ -1,3 +1,5 @@
+SET GLOBAL innodb_file_per_table = ON;
+set global innodb_compression_algorithm = 1;
 create database enctests;
 use enctests;
 create table t1(a int not null primary key, b char(200)) engine=innodb;
diff --git a/mysql-test/suite/encryption/t/innodb-first-page-read.test b/mysql-test/suite/encryption/t/innodb-first-page-read.test
index c86e16c52b8..4e6eeb25080 100644
--- a/mysql-test/suite/encryption/t/innodb-first-page-read.test
+++ b/mysql-test/suite/encryption/t/innodb-first-page-read.test
@@ -77,7 +77,13 @@ SELECT VARIABLE_VALUE <= 29  FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABL
 set global innodb_encrypt_tables=OFF;
 
 --echo # wait until tables are decrypted
---let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0
+SET GLOBAL innodb_max_dirty_pages_pct=0.0;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm=0.0;
+
+let $wait_condition =
+SELECT variable_value = 0
+FROM information_schema.global_status
+WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY';
 --source include/wait_condition.inc
 
 --echo # result should be actual number of tables except remote tables could be read twice
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index b045f8b16b8..9f9ed51a07f 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -1421,10 +1421,12 @@ fil_space_create(
 		encryption threads. */
 		fil_system.default_encrypt_tables.push_back(*space);
 		space->is_in_default_encrypt = true;
-		mutex_exit(&fil_system.mutex);
+	}
+
+	mutex_exit(&fil_system.mutex);
+
+	if (rotate && srv_n_fil_crypt_threads_started) {
 		os_event_set(fil_crypt_threads_event);
-	} else {
-		mutex_exit(&fil_system.mutex);
 	}
 
 	return(space);

From e46f76c9749d7758765ba274a212cfc2dcf3eeb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Mon, 21 Jun 2021 12:34:07 +0300
Subject: [PATCH 25/25] MDEV-15912: Remove traces of insert_undo

Let us simply refuse an upgrade from earlier versions if the
upgrade procedure was not followed. This simplifies the purge,
commit, and rollback of transactions.

Before upgrading to MariaDB 10.3 or later, a clean shutdown
of the server (with innodb_fast_shutdown=1 or 0) is necessary,
to ensure that any incomplete transactions are rolled back.
The undo log format was changed in MDEV-12288. There is only
one persistent undo log for each transaction.
---
 extra/innochecksum.cc                         |  39 +---
 .../suite/innodb_zip/r/innochecksum_3.result  |   8 +-
 storage/innobase/include/trx0purge.h          |  34 ++-
 storage/innobase/include/trx0rseg.h           |  39 ++--
 storage/innobase/include/trx0trx.h            |  13 +-
 storage/innobase/include/trx0undo.h           |  23 +-
 storage/innobase/lock/lock0lock.cc            |   2 +-
 storage/innobase/srv/srv0start.cc             |  21 +-
 storage/innobase/trx/trx0purge.cc             |  63 +++---
 storage/innobase/trx/trx0roll.cc              |  40 +---
 storage/innobase/trx/trx0rseg.cc              |  78 +++----
 storage/innobase/trx/trx0trx.cc               |  71 ++-----
 storage/innobase/trx/trx0undo.cc              | 197 ++++++++----------
 13 files changed, 248 insertions(+), 380 deletions(-)

diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc
index 73f62c26dd2..583a0db04f9 100644
--- a/extra/innochecksum.cc
+++ b/extra/innochecksum.cc
@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
-   Copyright (c) 2014, 2019, MariaDB Corporation.
+   Copyright (c) 2014, 2021, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -138,13 +138,10 @@ static ulong			write_check;
 struct innodb_page_type {
 	int n_undo_state_active;
 	int n_undo_state_cached;
-	int n_undo_state_to_free;
 	int n_undo_state_to_purge;
 	int n_undo_state_prepared;
 	int n_undo_state_other;
-	int n_undo_insert;
-	int n_undo_update;
-	int n_undo_other;
+	int n_undo;
 	int n_fil_page_index;
 	int n_fil_page_undo_log;
 	int n_fil_page_inode;
@@ -955,21 +952,7 @@ parse_page(
 			fprintf(file, "#::%llu\t\t|\t\tUndo log page\t\t\t|",
 				cur_page_num);
 		}
-		if (undo_page_type == TRX_UNDO_INSERT) {
-			page_type.n_undo_insert++;
-			if (page_type_dump) {
-				fprintf(file, "\t%s",
-					"Insert Undo log page");
-			}
-
-		} else if (undo_page_type == TRX_UNDO_UPDATE) {
-			page_type.n_undo_update++;
-			if (page_type_dump) {
-				fprintf(file, "\t%s",
-					"Update undo log page");
-			}
-		}
-
+		page_type.n_undo++;
 		undo_page_type = mach_read_from_2(page + TRX_UNDO_SEG_HDR +
 						  TRX_UNDO_STATE);
 		switch (undo_page_type) {
@@ -989,14 +972,6 @@ parse_page(
 				}
 				break;
 
-			case TRX_UNDO_TO_FREE:
-				page_type.n_undo_state_to_free++;
-				if (page_type_dump) {
-					fprintf(file, ", %s", "Insert undo "
-						"segment that can be freed");
-				}
-				break;
-
 			case TRX_UNDO_TO_PURGE:
 				page_type.n_undo_state_to_purge++;
 				if (page_type_dump) {
@@ -1220,15 +1195,11 @@ print_summary(
 
 	fprintf(fil_out, "\n===============================================\n");
 	fprintf(fil_out, "Additional information:\n");
-	fprintf(fil_out, "Undo page type: %d insert, %d update, %d other\n",
-		page_type.n_undo_insert,
-		page_type.n_undo_update,
-		page_type.n_undo_other);
-	fprintf(fil_out, "Undo page state: %d active, %d cached, %d to_free, %d"
+	fprintf(fil_out, "Undo page type: %d\n", page_type.n_undo);
+	fprintf(fil_out, "Undo page state: %d active, %d cached, %d"
 		" to_purge, %d prepared, %d other\n",
 		page_type.n_undo_state_active,
 		page_type.n_undo_state_cached,
-		page_type.n_undo_state_to_free,
 		page_type.n_undo_state_to_purge,
 		page_type.n_undo_state_prepared,
 		page_type.n_undo_state_other);
diff --git a/mysql-test/suite/innodb_zip/r/innochecksum_3.result b/mysql-test/suite/innodb_zip/r/innochecksum_3.result
index aaab68b3df9..830888e6b8a 100644
--- a/mysql-test/suite/innodb_zip/r/innochecksum_3.result
+++ b/mysql-test/suite/innodb_zip/r/innochecksum_3.result
@@ -109,8 +109,8 @@ File::tab#.ibd
 
 ===============================================
 Additional information:
-Undo page type: # insert, # update, # other
-Undo page state: # active, # cached, # to_free, # to_purge, # prepared, # other
+Undo page type: #
+Undo page state: # active, # cached, # to_purge, # prepared, # other
 index_id	#pages		#leaf_pages	#recs_per_page	#bytes_per_page
 #		#		#		#		#
 #		#		#		#		#
@@ -144,8 +144,8 @@ File::tab#.ibd
 
 ===============================================
 Additional information:
-Undo page type: # insert, # update, # other
-Undo page state: # active, # cached, # to_free, # to_purge, # prepared, # other
+Undo page type: #
+Undo page state: # active, # cached, # to_purge, # prepared, # other
 index_id	#pages		#leaf_pages	#recs_per_page	#bytes_per_page
 #		#		#		#		#
 #		#		#		#		#
diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h
index 4bc5aded341..f45bd1410f5 100644
--- a/storage/innobase/include/trx0purge.h
+++ b/storage/innobase/include/trx0purge.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2018, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -78,20 +78,16 @@ public:
 	typedef trx_rsegs_t::iterator iterator;
 	typedef trx_rsegs_t::const_iterator const_iterator;
 
-	/** Default constructor */
-	TrxUndoRsegs() {}
+	TrxUndoRsegs() : trx_no(0), m_rsegs() {}
 	/** Constructor */
 	TrxUndoRsegs(trx_rseg_t& rseg)
-		: m_commit(rseg.last_commit), m_rsegs(1, &rseg) {}
+		: trx_no(rseg.last_trx_no()), m_rsegs(1, &rseg) {}
 	/** Constructor */
 	TrxUndoRsegs(trx_id_t trx_no, trx_rseg_t& rseg)
-		: m_commit(trx_no << 1), m_rsegs(1, &rseg) {}
-
-	/** @return the transaction commit identifier */
-	trx_id_t trx_no() const { return m_commit >> 1; }
+		: trx_no(trx_no), m_rsegs(1, &rseg) {}
 
 	bool operator!=(const TrxUndoRsegs& other) const
-	{ return m_commit != other.m_commit; }
+	{ return trx_no != other.trx_no; }
 	bool empty() const { return m_rsegs.empty(); }
 	void erase(iterator& it) { m_rsegs.erase(it); }
 	iterator begin() { return(m_rsegs.begin()); }
@@ -105,14 +101,14 @@ public:
 	@return true if elem1 > elem2 else false.*/
 	bool operator()(const TrxUndoRsegs& lhs, const TrxUndoRsegs& rhs)
 	{
-		return(lhs.m_commit > rhs.m_commit);
+		return(lhs.trx_no > rhs.trx_no);
 	}
 
+	/** Copy of trx_rseg_t::last_trx_no() */
+	trx_id_t trx_no;
 private:
-	/** Copy trx_rseg_t::last_commit */
-	trx_id_t		m_commit;
 	/** Rollback segments of a transaction, scheduled for purge. */
-	trx_rsegs_t		m_rsegs;
+	trx_rsegs_t m_rsegs;
 };
 
 typedef std::priority_queue<
@@ -370,17 +366,13 @@ public:
 	{
 		bool operator<=(const iterator& other) const
 		{
-			if (commit < other.commit) return true;
-			if (commit > other.commit) return false;
+			if (trx_no < other.trx_no) return true;
+			if (trx_no > other.trx_no) return false;
 			return undo_no <= other.undo_no;
 		}
 
-		/** @return the commit number of the transaction */
-		trx_id_t trx_no() const { return commit >> 1; }
-		void reset_trx_no(trx_id_t trx_no) { commit = trx_no << 1; }
-
-		/** 2 * trx_t::no + old_insert of the committed transaction */
-		trx_id_t	commit;
+		/** trx_t::no of the committed transaction */
+		trx_id_t	trx_no;
 		/** The record number within the committed transaction's undo
 		log, increasing, purged from from 0 onwards */
 		undo_no_t	undo_no;
diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h
index d4fdb19a988..687c6fa0e97 100644
--- a/storage/innobase/include/trx0rseg.h
+++ b/storage/innobase/include/trx0rseg.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2019, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -82,9 +82,8 @@ trx_rseg_header_create(
 	buf_block_t*	sys_header,
 	mtr_t*		mtr);
 
-/** Initialize the rollback segments in memory at database startup. */
-void
-trx_rseg_array_init();
+/** Initialize or recover the rollback segments at startup. */
+dberr_t trx_rseg_array_init();
 
 /** Free a rollback segment in memory. */
 void
@@ -147,21 +146,13 @@ struct trx_rseg_t {
 	/** List of undo log segments cached for fast reuse */
 	UT_LIST_BASE_NODE_T(trx_undo_t)	undo_cached;
 
-	/** List of recovered old insert_undo logs of incomplete
-	transactions (to roll back or XA COMMIT & purge) */
-	UT_LIST_BASE_NODE_T(trx_undo_t) old_insert_list;
-
 	/*--------------------------------------------------------*/
 
-	/** Page number of the last not yet purged log header in the history
-	list; FIL_NULL if all list purged */
-	ulint				last_page_no;
+  /** Last not yet purged undo log header; FIL_NULL if all purged */
+  uint32_t last_page_no;
 
-	/** Byte offset of the last not yet purged log header */
-	ulint				last_offset;
-
-	/** trx_t::no * 2 + old_insert of the last not yet purged log */
-	trx_id_t			last_commit;
+  /** trx_t::no | last_offset << 48 */
+  uint64_t last_commit_and_offset;
 
 	/** Whether the log segment needs purge */
 	bool				needs_purge;
@@ -173,13 +164,17 @@ struct trx_rseg_t {
 	UNDO-tablespace marked for truncate. */
 	bool				skip_allocation;
 
-	/** @return the commit ID of the last committed transaction */
-	trx_id_t last_trx_no() const { return last_commit >> 1; }
+  /** @return the commit ID of the last committed transaction */
+  trx_id_t last_trx_no() const
+  { return last_commit_and_offset & ((1ULL << 48) - 1); }
+  /** @return header offset of the last committed transaction */
+  uint16_t last_offset() const
+  { return static_cast<uint16_t>(last_commit_and_offset >> 48); }
 
-	void set_last_trx_no(trx_id_t trx_no, bool is_update)
-	{
-		last_commit = trx_no << 1 | trx_id_t(is_update);
-	}
+  void set_last_commit(ulint last_offset, trx_id_t trx_no)
+  {
+    last_commit_and_offset= static_cast<uint64_t>(last_offset) << 48 | trx_no;
+  }
 
 	/** @return whether the rollback segment is persistent */
 	bool is_persistent() const
diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h
index 7cd11bf5496..b4ed45f3565 100644
--- a/storage/innobase/include/trx0trx.h
+++ b/storage/innobase/include/trx0trx.h
@@ -79,7 +79,7 @@ void trx_free_at_shutdown(trx_t *trx);
 void trx_disconnect_prepared(trx_t *trx);
 
 /** Initialize (resurrect) transactions at startup. */
-void trx_lists_init_at_db_start();
+dberr_t trx_lists_init_at_db_start();
 
 /*************************************************************//**
 Starts the transaction if it is not yet started. */
@@ -698,10 +698,6 @@ struct trx_undo_ptr_t {
 					yet */
 	trx_undo_t*	undo;		/*!< pointer to the undo log, or
 					NULL if nothing logged yet */
-	trx_undo_t*     old_insert;	/*!< pointer to recovered
-					insert undo log, or NULL if no
-					INSERT transactions were
-					recovered from old-format undo logs */
 };
 
 /** An instance of temporary rollback segment. */
@@ -1055,13 +1051,6 @@ public:
 		return(has_logged_persistent() || rsegs.m_noredo.undo);
 	}
 
-	/** @return whether any undo log has been generated or
-	recovered */
-	bool has_logged_or_recovered() const
-	{
-		return(has_logged() || rsegs.m_redo.old_insert);
-	}
-
 	/** @return rollback segment for modifying temporary tables */
 	trx_rseg_t* get_temp_rseg()
 	{
diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h
index 22420f111b5..b0b7ce2941f 100644
--- a/storage/innobase/include/trx0undo.h
+++ b/storage/innobase/include/trx0undo.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2020, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -253,13 +253,11 @@ trx_undo_set_state_at_prepare(
 	bool		rollback,
 	mtr_t*		mtr);
 
-/** Free an old insert or temporary undo log after commit or rollback.
+/** Free temporary undo log after commit or rollback.
 The information is not needed after a commit or rollback, therefore
 the data can be discarded.
-@param[in,out]	undo	undo log
-@param[in]	is_temp	whether this is temporary undo log */
-void
-trx_undo_commit_cleanup(trx_undo_t* undo, bool is_temp);
+@param undo     temporary undo log */
+void trx_undo_commit_cleanup(trx_undo_t *undo);
 
 /** At shutdown, frees the undo logs of a transaction. */
 void
@@ -302,10 +300,11 @@ trx_undo_parse_page_header(
 @param[in]	id		rollback segment slot
 @param[in]	page_no		undo log segment page number
 @param[in,out]	max_trx_id	the largest observed transaction ID
-@return	size of the undo log in pages */
-ulint
-trx_undo_mem_create_at_db_start(trx_rseg_t* rseg, ulint id, ulint page_no,
-				trx_id_t& max_trx_id);
+@return	the undo log
+@retval nullptr on error */
+trx_undo_t *
+trx_undo_mem_create_at_db_start(trx_rseg_t *rseg, ulint id, uint32_t page_no,
+                                trx_id_t &max_trx_id);
 
 #endif /* !UNIV_INNOCHECKSUM */
 
@@ -319,7 +318,6 @@ trx_undo_mem_create_at_db_start(trx_rseg_t* rseg, ulint id, ulint page_no,
 #define TRX_UNDO_ACTIVE		1	/* contains an undo log of an active
 					transaction */
 #define	TRX_UNDO_CACHED		2	/* cached for quick reuse */
-#define	TRX_UNDO_TO_FREE	3	/* insert undo segment can be freed */
 #define	TRX_UNDO_TO_PURGE	4	/* update undo segment will not be
 					reused: it can be freed in purge when
 					all undo data in it is removed */
@@ -383,7 +381,8 @@ struct trx_undo_t {
 /** Transaction undo log page header offsets */
 /* @{ */
 #define	TRX_UNDO_PAGE_TYPE	0	/*!< unused; 0 (before MariaDB 10.3.1:
-					TRX_UNDO_INSERT or TRX_UNDO_UPDATE) */
+					1=TRX_UNDO_INSERT or
+					2=TRX_UNDO_UPDATE) */
 #define	TRX_UNDO_PAGE_START	2	/*!< Byte offset where the undo log
 					records for the LATEST transaction
 					start on this page (remember that
diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc
index b3086842624..c584de0fa5d 100644
--- a/storage/innobase/lock/lock0lock.cc
+++ b/storage/innobase/lock/lock0lock.cc
@@ -4584,7 +4584,7 @@ lock_print_info_summary(
 		"Purge done for trx's n:o < " TRX_ID_FMT
 		" undo n:o < " TRX_ID_FMT " state: %s\n"
 		"History list length %u\n",
-		purge_sys.tail.trx_no(),
+		purge_sys.tail.trx_no,
 		purge_sys.tail.undo_no,
 		purge_sys.enabled()
 		? (purge_sys.running() ? "running"
diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc
index 7314fd60cd6..7d5679e889e 100644
--- a/storage/innobase/srv/srv0start.cc
+++ b/storage/innobase/srv/srv0start.cc
@@ -1404,6 +1404,11 @@ dberr_t srv_start(bool create_new_db)
 	      || is_mariabackup_restore_or_export());
 
 
+	if (srv_force_recovery) {
+		ib::info() << "!!! innodb_force_recovery is set to "
+			<< srv_force_recovery << " !!!";
+	}
+
 	if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) {
 		srv_read_only_mode = true;
 	}
@@ -1922,7 +1927,11 @@ files_checked:
 		All the remaining rollback segments will be created later,
 		after the double write buffer has been created. */
 		trx_sys_create_sys_pages();
-		trx_lists_init_at_db_start();
+		err = trx_lists_init_at_db_start();
+
+		if (err != DB_SUCCESS) {
+			return(srv_init_abort(err));
+		}
 
 		err = dict_create();
 
@@ -1986,7 +1995,10 @@ files_checked:
 		case SRV_OPERATION_RESTORE:
 			/* This must precede
 			recv_apply_hashed_log_recs(true). */
-			trx_lists_init_at_db_start();
+			err = trx_lists_init_at_db_start();
+			if (err != DB_SUCCESS) {
+				return srv_init_abort(err);
+			}
 			break;
 		case SRV_OPERATION_RESTORE_DELTA:
 		case SRV_OPERATION_BACKUP:
@@ -2453,11 +2465,6 @@ skip_monitors:
 			   << "; transaction id " << trx_sys.get_max_trx_id();
 	}
 
-	if (srv_force_recovery > 0) {
-		ib::info() << "!!! innodb_force_recovery is set to "
-			<< srv_force_recovery << " !!!";
-	}
-
 	if (srv_force_recovery == 0) {
 		/* In the insert buffer we may have even bigger tablespace
 		id's, because we may have dropped those tablespaces, but
diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc
index 02a524d6850..59d60d204d8 100644
--- a/storage/innobase/trx/trx0purge.cc
+++ b/storage/innobase/trx/trx0purge.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2020, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -87,7 +87,7 @@ inline bool TrxUndoRsegsIterator::set_next()
 		number shouldn't increase. Undo the increment of
 		expected commit done by caller assuming rollback
 		segments from given transaction are done. */
-		purge_sys.tail.commit = (*m_iter)->last_commit;
+		purge_sys.tail.trx_no = (*m_iter)->last_trx_no();
 	} else if (!purge_sys.purge_queue.empty()) {
 		m_rsegs = purge_sys.purge_queue.top();
 		purge_sys.purge_queue.pop();
@@ -108,17 +108,17 @@ inline bool TrxUndoRsegsIterator::set_next()
 	mutex_enter(&purge_sys.rseg->mutex);
 
 	ut_a(purge_sys.rseg->last_page_no != FIL_NULL);
-	ut_ad(purge_sys.rseg->last_trx_no() == m_rsegs.trx_no());
+	ut_ad(purge_sys.rseg->last_trx_no() == m_rsegs.trx_no);
 
 	/* We assume in purge of externally stored fields that space id is
 	in the range of UNDO tablespace space ids */
 	ut_ad(purge_sys.rseg->space->id == TRX_SYS_SPACE
 	      || srv_is_undo_tablespace(purge_sys.rseg->space->id));
 
-	ut_a(purge_sys.tail.commit <= purge_sys.rseg->last_commit);
+	ut_a(purge_sys.tail.trx_no <= purge_sys.rseg->last_trx_no());
 
-	purge_sys.tail.commit = purge_sys.rseg->last_commit;
-	purge_sys.hdr_offset = purge_sys.rseg->last_offset;
+	purge_sys.tail.trx_no = purge_sys.rseg->last_trx_no();
+	purge_sys.hdr_offset = purge_sys.rseg->last_offset();
 	purge_sys.hdr_page_no = purge_sys.rseg->last_page_no;
 
 	mutex_exit(&purge_sys.rseg->mutex);
@@ -209,8 +209,7 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr)
 {
 	DBUG_PRINT("trx", ("commit(" TRX_ID_FMT "," TRX_ID_FMT ")",
 			   trx->id, trx->no));
-	ut_ad(undo == trx->rsegs.m_redo.undo
-	      || undo == trx->rsegs.m_redo.old_insert);
+	ut_ad(undo == trx->rsegs.m_redo.undo);
 	trx_rseg_t*	rseg		= trx->rsegs.m_redo.rseg;
 	ut_ad(undo->rseg == rseg);
 	trx_rsegf_t*	rseg_header	= trx_rsegf_get(
@@ -302,9 +301,8 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr)
 	}
 
 	if (rseg->last_page_no == FIL_NULL) {
-		rseg->last_page_no = undo->hdr_page_no;
-		rseg->last_offset = undo->hdr_offset;
-		rseg->set_last_trx_no(trx->no, undo == trx->rsegs.m_redo.undo);
+		rseg->last_page_no = static_cast<uint32_t>(undo->hdr_page_no);
+		rseg->set_last_commit(undo->hdr_offset, trx->no);
 		rseg->needs_purge = true;
 	}
 
@@ -460,8 +458,8 @@ func_exit:
 
 	undo_trx_no = mach_read_from_8(log_hdr + TRX_UNDO_TRX_NO);
 
-	if (undo_trx_no >= limit.trx_no()) {
-		if (undo_trx_no == limit.trx_no()) {
+	if (undo_trx_no >= limit.trx_no) {
+		if (undo_trx_no == limit.trx_no) {
 			trx_undo_truncate_start(
 				&rseg, hdr_addr.page,
 				hdr_addr.boffset, limit.undo_no);
@@ -884,7 +882,7 @@ trx_purge_initiate_truncate(
 			     undo != NULL && all_free;
 			     undo = UT_LIST_GET_NEXT(undo_list, undo)) {
 
-				if (limit.trx_no() < undo->trx_id) {
+				if (limit.trx_no < undo->trx_id) {
 					all_free = false;
 				} else {
 					cached_undo_size += undo->size;
@@ -986,7 +984,6 @@ not_found:
 		/* Before re-initialization ensure that we free the existing
 		structure. There can't be any active transactions. */
 		ut_a(UT_LIST_GET_LEN(rseg->undo_list) == 0);
-		ut_a(UT_LIST_GET_LEN(rseg->old_insert_list) == 0);
 
 		trx_undo_t*	next_undo;
 
@@ -1002,7 +999,6 @@ not_found:
 
 		UT_LIST_INIT(rseg->undo_list, &trx_undo_t::undo_list);
 		UT_LIST_INIT(rseg->undo_cached, &trx_undo_t::undo_list);
-		UT_LIST_INIT(rseg->old_insert_list, &trx_undo_t::undo_list);
 
 		/* These were written by trx_rseg_header_create(). */
 		ut_ad(!mach_read_from_4(TRX_RSEG + TRX_RSEG_FORMAT
@@ -1014,8 +1010,7 @@ not_found:
 		rseg->curr_size = 1;
 		rseg->trx_ref_count = 0;
 		rseg->last_page_no = FIL_NULL;
-		rseg->last_offset = 0;
-		rseg->last_commit = 0;
+		rseg->last_commit_and_offset = 0;
 		rseg->needs_purge = false;
 	}
 
@@ -1076,12 +1071,12 @@ function is called, the caller must not have any latches on undo log pages!
 static void trx_purge_truncate_history()
 {
 	ut_ad(purge_sys.head <= purge_sys.tail);
-	purge_sys_t::iterator& head = purge_sys.head.commit
+	purge_sys_t::iterator& head = purge_sys.head.trx_no
 		? purge_sys.head : purge_sys.tail;
 
-	if (head.trx_no() >= purge_sys.view.low_limit_no()) {
+	if (head.trx_no >= purge_sys.view.low_limit_no()) {
 		/* This is sometimes necessary. TODO: find out why. */
-		head.reset_trx_no(purge_sys.view.low_limit_no());
+		head.trx_no = purge_sys.view.low_limit_no();
 		head.undo_no = 0;
 	}
 
@@ -1109,7 +1104,6 @@ static void trx_purge_rseg_get_next_history_log(
 					handled */
 {
 	page_t*		undo_page;
-	trx_ulogf_t*	log_hdr;
 	fil_addr_t	prev_log_addr;
 	trx_id_t	trx_no;
 	mtr_t		mtr;
@@ -1118,7 +1112,7 @@ static void trx_purge_rseg_get_next_history_log(
 
 	ut_a(purge_sys.rseg->last_page_no != FIL_NULL);
 
-	purge_sys.tail.commit = purge_sys.rseg->last_commit + 1;
+	purge_sys.tail.trx_no = purge_sys.rseg->last_trx_no() + 1;
 	purge_sys.tail.undo_no = 0;
 	purge_sys.next_stored = false;
 
@@ -1128,7 +1122,7 @@ static void trx_purge_rseg_get_next_history_log(
 		page_id_t(purge_sys.rseg->space->id,
 			  purge_sys.rseg->last_page_no), &mtr);
 
-	log_hdr = undo_page + purge_sys.rseg->last_offset;
+	const trx_ulogf_t* log_hdr = undo_page + purge_sys.rseg->last_offset();
 
 	/* Increase the purge page count by one for every handled log */
 
@@ -1160,17 +1154,16 @@ static void trx_purge_rseg_get_next_history_log(
 		+ prev_log_addr.boffset;
 
 	trx_no = mach_read_from_8(log_hdr + TRX_UNDO_TRX_NO);
-	unsigned purge = mach_read_from_2(log_hdr + TRX_UNDO_NEEDS_PURGE);
-	ut_ad(purge <= 1);
+	ut_ad(mach_read_from_2(log_hdr + TRX_UNDO_NEEDS_PURGE) <= 1);
 
 	mtr_commit(&mtr);
 
 	mutex_enter(&purge_sys.rseg->mutex);
 
-	purge_sys.rseg->last_page_no = prev_log_addr.page;
-	purge_sys.rseg->last_offset = prev_log_addr.boffset;
-	purge_sys.rseg->set_last_trx_no(trx_no, purge != 0);
-	purge_sys.rseg->needs_purge = purge != 0;
+	purge_sys.rseg->last_page_no = static_cast<uint32_t>(
+		prev_log_addr.page);
+	purge_sys.rseg->set_last_commit(prev_log_addr.boffset, trx_no);
+	purge_sys.rseg->needs_purge = log_hdr[TRX_UNDO_NEEDS_PURGE + 1] != 0;
 
 	/* Purge can also produce events, however these are already ordered
 	in the rollback segment and any user generated event will be greater
@@ -1187,15 +1180,13 @@ static void trx_purge_rseg_get_next_history_log(
 }
 
 /** Position the purge sys "iterator" on the undo record to use for purging. */
-static
-void
-trx_purge_read_undo_rec()
+static void trx_purge_read_undo_rec()
 {
 	ulint		offset;
 	ulint		page_no;
 	ib_uint64_t	undo_no;
 
-	purge_sys.hdr_offset = purge_sys.rseg->last_offset;
+	purge_sys.hdr_offset = purge_sys.rseg->last_offset();
 	page_no = purge_sys.hdr_page_no = purge_sys.rseg->last_page_no;
 
 	if (purge_sys.rseg->needs_purge) {
@@ -1268,7 +1259,7 @@ trx_purge_get_next_rec(
 	mtr_t		mtr;
 
 	ut_ad(purge_sys.next_stored);
-	ut_ad(purge_sys.tail.trx_no() < purge_sys.view.low_limit_no());
+	ut_ad(purge_sys.tail.trx_no < purge_sys.view.low_limit_no());
 
 	space = purge_sys.rseg->space->id;
 	page_no = purge_sys.page_no;
@@ -1361,7 +1352,7 @@ trx_purge_fetch_next_rec(
 		}
 	}
 
-	if (purge_sys.tail.trx_no() >= purge_sys.view.low_limit_no()) {
+	if (purge_sys.tail.trx_no >= purge_sys.view.low_limit_no()) {
 
 		return(NULL);
 	}
diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc
index 88ff251548c..bffd168359b 100644
--- a/storage/innobase/trx/trx0roll.cc
+++ b/storage/innobase/trx/trx0roll.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2016, 2020, MariaDB Corporation.
+Copyright (c) 2016, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -70,12 +70,6 @@ static bool trx_rollback_finish(trx_t* trx)
 		ut_a(!srv_undo_sources);
 		ut_ad(srv_fast_shutdown);
 		ut_d(trx->in_rollback = false);
-		if (trx_undo_t*& undo = trx->rsegs.m_redo.old_insert) {
-			UT_LIST_REMOVE(trx->rsegs.m_redo.rseg->old_insert_list,
-				       undo);
-			ut_free(undo);
-			undo = NULL;
-		}
 		if (trx_undo_t*& undo = trx->rsegs.m_redo.undo) {
 			UT_LIST_REMOVE(trx->rsegs.m_redo.rseg->undo_list,
 				       undo);
@@ -124,7 +118,7 @@ trx_rollback_to_savepoint_low(
 
 	trx->error_state = DB_SUCCESS;
 
-	if (trx->has_logged_or_recovered()) {
+	if (trx->has_logged()) {
 
 		ut_ad(trx->rsegs.m_redo.rseg != 0
 		      || trx->rsegs.m_noredo.rseg != 0);
@@ -240,7 +234,7 @@ dberr_t trx_rollback_for_mysql(trx_t* trx)
 	case TRX_STATE_PREPARED:
 	case TRX_STATE_PREPARED_RECOVERED:
 		ut_ad(!trx_is_autocommit_non_locking(trx));
-		if (trx->rsegs.m_redo.undo || trx->rsegs.m_redo.old_insert) {
+		if (trx->rsegs.m_redo.undo) {
 			/* The XA ROLLBACK of a XA PREPARE transaction
 			will consist of multiple mini-transactions.
 
@@ -256,11 +250,7 @@ dberr_t trx_rollback_for_mysql(trx_t* trx)
 			killed, and finally, the transaction would be
 			recovered in XA PREPARE state, with some of
 			the actions already having been rolled back. */
-			ut_ad(!trx->rsegs.m_redo.undo
-			      || trx->rsegs.m_redo.undo->rseg
-			      == trx->rsegs.m_redo.rseg);
-			ut_ad(!trx->rsegs.m_redo.old_insert
-			      || trx->rsegs.m_redo.old_insert->rseg
+			ut_ad(trx->rsegs.m_redo.undo->rseg
 			      == trx->rsegs.m_redo.rseg);
 			mtr_t		mtr;
 			mtr.start();
@@ -269,10 +259,6 @@ dberr_t trx_rollback_for_mysql(trx_t* trx)
 				trx_undo_set_state_at_prepare(trx, undo, true,
 							      &mtr);
 			}
-			if (trx_undo_t* undo = trx->rsegs.m_redo.old_insert) {
-				trx_undo_set_state_at_prepare(trx, undo, true,
-							      &mtr);
-			}
 			mutex_exit(&trx->rsegs.m_redo.rseg->mutex);
 			/* Write the redo log for the XA ROLLBACK
 			state change to the global buffer. It is
@@ -959,23 +945,13 @@ trx_roll_pop_top_rec_of_trx(trx_t* trx, roll_ptr_t* roll_ptr, mem_heap_t* heap)
 	}
 
 	trx_undo_t*	undo	= NULL;
-	trx_undo_t*	insert	= trx->rsegs.m_redo.old_insert;
 	trx_undo_t*	update	= trx->rsegs.m_redo.undo;
 	trx_undo_t*	temp	= trx->rsegs.m_noredo.undo;
 	const undo_no_t	limit	= trx->roll_limit;
 
-	ut_ad(!insert || !update || insert->empty() || update->empty()
-	      || insert->top_undo_no != update->top_undo_no);
-	ut_ad(!insert || !temp || insert->empty() || temp->empty()
-	      || insert->top_undo_no != temp->top_undo_no);
 	ut_ad(!update || !temp || update->empty() || temp->empty()
 	      || update->top_undo_no != temp->top_undo_no);
 
-	if (UNIV_LIKELY_NULL(insert)
-	    && !insert->empty() && limit <= insert->top_undo_no) {
-		undo = insert;
-	}
-
 	if (update && !update->empty() && update->top_undo_no >= limit) {
 		if (!undo) {
 			undo = update;
@@ -1020,18 +996,12 @@ trx_roll_pop_top_rec_of_trx(trx_t* trx, roll_ptr_t* roll_ptr, mem_heap_t* heap)
 		MDEV-12288 removed the insert_undo log. There is no
 		instant ADD COLUMN for temporary tables. Therefore,
 		this record can only be present in the main undo log. */
-		ut_ad(undo == update);
 		/* fall through */
 	case TRX_UNDO_RENAME_TABLE:
-		ut_ad(undo == insert || undo == update);
+		ut_ad(undo == update);
 		/* fall through */
 	case TRX_UNDO_INSERT_REC:
-		ut_ad(undo == insert || undo == update || undo == temp);
 		*roll_ptr |= 1ULL << ROLL_PTR_INSERT_FLAG_POS;
-		break;
-	default:
-		ut_ad(undo == update || undo == temp);
-		break;
 	}
 
 	trx->undo_no = undo_no;
diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc
index d6720979716..f660b3151e6 100644
--- a/storage/innobase/trx/trx0rseg.cc
+++ b/storage/innobase/trx/trx0rseg.cc
@@ -359,7 +359,6 @@ trx_rseg_mem_free(trx_rseg_t* rseg)
 
 	/* There can't be any active transactions. */
 	ut_a(UT_LIST_GET_LEN(rseg->undo_list) == 0);
-	ut_a(UT_LIST_GET_LEN(rseg->old_insert_list) == 0);
 
 	for (undo = UT_LIST_GET_FIRST(rseg->undo_cached);
 	     undo != NULL;
@@ -399,45 +398,45 @@ trx_rseg_mem_create(ulint id, fil_space_t* space, ulint page_no)
 		     &rseg->mutex);
 
 	UT_LIST_INIT(rseg->undo_list, &trx_undo_t::undo_list);
-	UT_LIST_INIT(rseg->old_insert_list, &trx_undo_t::undo_list);
 	UT_LIST_INIT(rseg->undo_cached, &trx_undo_t::undo_list);
 
 	return(rseg);
 }
 
 /** Read the undo log lists.
-@param[in,out]	rseg		rollback segment
-@param[in,out]	max_trx_id	maximum observed transaction identifier
-@param[in]	rseg_header	rollback segment header
-@return the combined size of undo log segments in pages */
-static
-ulint
-trx_undo_lists_init(trx_rseg_t* rseg, trx_id_t& max_trx_id,
-		    const trx_rsegf_t* rseg_header)
+@param[in,out]  rseg            rollback segment
+@param[in,out]  max_trx_id      maximum observed transaction identifier
+@param[in]      rseg_header     rollback segment header
+@return error code */
+static dberr_t trx_undo_lists_init(trx_rseg_t *rseg, trx_id_t &max_trx_id,
+                                   const trx_rsegf_t *rseg_header)
 {
 	ut_ad(srv_force_recovery < SRV_FORCE_NO_UNDO_LOG_SCAN);
 
-	ulint size = 0;
+  for (ulint i= 0; i < TRX_RSEG_N_SLOTS; i++)
+  {
+    uint32_t page_no= trx_rsegf_get_nth_undo(rseg_header, i);
+    if (page_no != FIL_NULL)
+    {
+      const trx_undo_t *undo= trx_undo_mem_create_at_db_start(rseg, i, page_no,
+                                                              max_trx_id);
+      if (!undo)
+        return DB_CORRUPTION;
+      rseg->curr_size+= undo->size;
+      MONITOR_INC(MONITOR_NUM_UNDO_SLOT_USED);
+    }
+  }
 
-	for (ulint i = 0; i < TRX_RSEG_N_SLOTS; i++) {
-		ulint	page_no = trx_rsegf_get_nth_undo(rseg_header, i);
-		if (page_no != FIL_NULL) {
-			size += trx_undo_mem_create_at_db_start(
-				rseg, i, page_no, max_trx_id);
-			MONITOR_INC(MONITOR_NUM_UNDO_SLOT_USED);
-		}
-	}
-
-	return(size);
+  return DB_SUCCESS;
 }
 
 /** Restore the state of a persistent rollback segment.
 @param[in,out]	rseg		persistent rollback segment
 @param[in,out]	max_trx_id	maximum observed transaction identifier
-@param[in,out]	mtr		mini-transaction */
-static
-void
-trx_rseg_mem_restore(trx_rseg_t* rseg, trx_id_t& max_trx_id, mtr_t* mtr)
+@param[in,out]	mtr		mini-transaction
+@return error code */
+static dberr_t trx_rseg_mem_restore(trx_rseg_t *rseg, trx_id_t &max_trx_id,
+                                    mtr_t *mtr)
 {
 	/* This is based on trx_rsegf_get_new().
 	We need to access buf_block_t. */
@@ -484,13 +483,16 @@ trx_rseg_mem_restore(trx_rseg_t* rseg, trx_id_t& max_trx_id, mtr_t* mtr)
 		/* mariabackup --prepare only deals with
 		the redo log and the data files, not with
 		transactions or the data dictionary. */
-		return;
+		return DB_SUCCESS;
 	}
 
 	/* Initialize the undo log lists according to the rseg header */
 
 	rseg->curr_size = mach_read_from_4(rseg_header + TRX_RSEG_HISTORY_SIZE)
-		+ 1 + trx_undo_lists_init(rseg, max_trx_id, rseg_header);
+		+ 1;
+	if (dberr_t err = trx_undo_lists_init(rseg, max_trx_id, rseg_header)) {
+		return err;
+	}
 
 	if (ulint len = flst_get_len(rseg_header + TRX_RSEG_HISTORY)) {
 		trx_sys.history_add(int32(len));
@@ -498,8 +500,7 @@ trx_rseg_mem_restore(trx_rseg_t* rseg, trx_id_t& max_trx_id, mtr_t* mtr)
 		fil_addr_t	node_addr = trx_purge_get_log_from_hist(
 			flst_get_last(rseg_header + TRX_RSEG_HISTORY, mtr));
 
-		rseg->last_page_no = node_addr.page;
-		rseg->last_offset = node_addr.boffset;
+		rseg->last_page_no = static_cast<uint32_t>(node_addr.page);
 
 		const trx_ulogf_t*	undo_log_hdr = trx_undo_page_get(
 			page_id_t(rseg->space->id, node_addr.page), mtr)
@@ -513,10 +514,10 @@ trx_rseg_mem_restore(trx_rseg_t* rseg, trx_id_t& max_trx_id, mtr_t* mtr)
 		if (id > max_trx_id) {
 			max_trx_id = id;
 		}
+		rseg->set_last_commit(node_addr.boffset, id);
 		unsigned purge = mach_read_from_2(
 			undo_log_hdr + TRX_UNDO_NEEDS_PURGE);
 		ut_ad(purge <= 1);
-		rseg->set_last_trx_no(id, purge != 0);
 		rseg->needs_purge = purge != 0;
 
 		if (rseg->last_page_no != FIL_NULL) {
@@ -526,6 +527,8 @@ trx_rseg_mem_restore(trx_rseg_t* rseg, trx_id_t& max_trx_id, mtr_t* mtr)
 			purge_sys.purge_queue.push(*rseg);
 		}
 	}
+
+	return DB_SUCCESS;
 }
 
 /** Read binlog metadata from the TRX_SYS page, in case we are upgrading
@@ -549,9 +552,8 @@ static void trx_rseg_init_binlog_info(const page_t* page)
 #endif
 }
 
-/** Initialize the rollback segments in memory at database startup. */
-void
-trx_rseg_array_init()
+/** Initialize or recover the rollback segments at startup. */
+dberr_t trx_rseg_array_init()
 {
 	trx_id_t max_trx_id = 0;
 
@@ -563,9 +565,9 @@ trx_rseg_array_init()
 	wsrep_sys_xid.null();
 	bool wsrep_xid_in_rseg_found = false;
 #endif
+	mtr_t mtr;
 
 	for (ulint rseg_id = 0; rseg_id < TRX_SYS_N_RSEGS; rseg_id++) {
-		mtr_t mtr;
 		mtr.start();
 		if (const buf_block_t* sys = trx_sysf_get(&mtr, false)) {
 			if (rseg_id == 0) {
@@ -593,7 +595,11 @@ trx_rseg_array_init()
 				ut_ad(rseg->id == rseg_id);
 				ut_ad(!trx_sys.rseg_array[rseg_id]);
 				trx_sys.rseg_array[rseg_id] = rseg;
-				trx_rseg_mem_restore(rseg, max_trx_id, &mtr);
+				if (dberr_t err = trx_rseg_mem_restore(
+					    rseg, max_trx_id, &mtr)) {
+					mtr.commit();
+					return err;
+				}
 #ifdef WITH_WSREP
 				if (!wsrep_sys_xid.is_null() &&
 				    !wsrep_sys_xid.eq(&trx_sys.recovered_wsrep_xid)) {
@@ -620,7 +626,6 @@ trx_rseg_array_init()
 		If no rollback segment has a WSREP XID set,
 		we must copy the XID found in TRX_SYS page
 		to rollback segments. */
-		mtr_t mtr;
 		mtr.start();
 
 		if (!wsrep_xid_in_rseg_found) {
@@ -638,6 +643,7 @@ trx_rseg_array_init()
 #endif
 
 	trx_sys.init_max_trx_id(max_trx_id + 1);
+	return DB_SUCCESS;
 }
 
 /** Create a persistent rollback segment.
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index 2620005269c..c20a1270bb9 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -664,8 +664,7 @@ trx_resurrect_table_locks(
 
 static void trx_resurrect(trx_undo_t *undo, trx_rseg_t *rseg,
                           time_t start_time, ulonglong start_time_micro,
-                          uint64_t *rows_to_undo,
-                          bool is_old_insert)
+                          uint64_t *rows_to_undo)
 {
   trx_state_t state;
   /*
@@ -688,8 +687,6 @@ static void trx_resurrect(trx_undo_t *undo, trx_rseg_t *rseg,
     state= TRX_STATE_PREPARED;
     break;
   default:
-    if (is_old_insert && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO)
-      trx_undo_commit_cleanup(undo, false);
     return;
   }
 
@@ -699,11 +696,7 @@ static void trx_resurrect(trx_undo_t *undo, trx_rseg_t *rseg,
   ut_d(trx->start_line= __LINE__);
   ut_ad(trx->no == TRX_ID_MAX);
 
-  if (is_old_insert)
-    trx->rsegs.m_redo.old_insert= undo;
-  else
-    trx->rsegs.m_redo.undo= undo;
-
+  trx->rsegs.m_redo.undo= undo;
   trx->undo_no= undo->top_undo_no + 1;
   trx->rsegs.m_redo.rseg= rseg;
   /*
@@ -734,8 +727,7 @@ static void trx_resurrect(trx_undo_t *undo, trx_rseg_t *rseg,
 
 
 /** Initialize (resurrect) transactions at startup. */
-void
-trx_lists_init_at_db_start()
+dberr_t trx_lists_init_at_db_start()
 {
 	ut_a(srv_is_being_started);
 	ut_ad(!srv_was_started);
@@ -744,16 +736,18 @@ trx_lists_init_at_db_start()
 		/* mariabackup --prepare only deals with
 		the redo log and the data files, not with
 		transactions or the data dictionary. */
-		trx_rseg_array_init();
-		return;
+		return trx_rseg_array_init();
 	}
 
 	if (srv_force_recovery >= SRV_FORCE_NO_UNDO_LOG_SCAN) {
-		return;
+		return DB_SUCCESS;
 	}
 
 	purge_sys.create();
-	trx_rseg_array_init();
+	if (dberr_t err = trx_rseg_array_init()) {
+		ib::info() << "Retry with innodb_force_recovery=5";
+		return err;
+	}
 
 	/* Look from the rollback segments if there exist undo logs for
 	transactions. */
@@ -771,17 +765,6 @@ trx_lists_init_at_db_start()
 		if (rseg == NULL) {
 			continue;
 		}
-
-		/* Resurrect transactions that were doing inserts
-		using the old separate insert_undo log. */
-		undo = UT_LIST_GET_FIRST(rseg->old_insert_list);
-		while (undo) {
-			trx_undo_t* next = UT_LIST_GET_NEXT(undo_list, undo);
-			trx_resurrect(undo, rseg, start_time, start_time_micro,
-				      &rows_to_undo, true);
-			undo = next;
-		}
-
 		/* Ressurrect other transactions. */
 		for (undo = UT_LIST_GET_FIRST(rseg->undo_list);
 		     undo != NULL;
@@ -789,8 +772,7 @@ trx_lists_init_at_db_start()
 			trx_t *trx = trx_sys.find(0, undo->trx_id, false);
 			if (!trx) {
 				trx_resurrect(undo, rseg, start_time,
-					      start_time_micro,
-					      &rows_to_undo, false);
+					      start_time_micro, &rows_to_undo);
 			} else {
 				ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) ||
 				      trx_state_eq(trx, TRX_STATE_PREPARED));
@@ -825,6 +807,7 @@ trx_lists_init_at_db_start()
 		ib::info() << "Trx id counter is " << trx_sys.get_max_trx_id();
 	}
 	trx_sys.clone_oldest_view();
+	return DB_SUCCESS;
 }
 
 /** Assign a persistent rollback segment in a round-robin fashion,
@@ -1121,30 +1104,22 @@ trx_write_serialisation_history(
 	trx_rseg_t*	rseg = trx->rsegs.m_redo.rseg;
 	if (!rseg) {
 		ut_ad(!trx->rsegs.m_redo.undo);
-		ut_ad(!trx->rsegs.m_redo.old_insert);
 		return;
 	}
 
 	trx_undo_t*& undo = trx->rsegs.m_redo.undo;
-	trx_undo_t*& old_insert = trx->rsegs.m_redo.old_insert;
 
-	if (!undo && !old_insert) {
+	if (!undo) {
 		return;
 	}
 
 	ut_ad(!trx->read_only);
 	ut_ad(!undo || undo->rseg == rseg);
-	ut_ad(!old_insert || old_insert->rseg == rseg);
 	mutex_enter(&rseg->mutex);
 
 	/* Assign the transaction serialisation number and add any
 	undo log to the purge queue. */
 	trx_serialise(trx);
-
-	if (UNIV_LIKELY_NULL(old_insert)) {
-		UT_LIST_REMOVE(rseg->old_insert_list, old_insert);
-		trx_purge_add_undo_to_history(trx, old_insert, mtr);
-	}
 	if (undo) {
 		UT_LIST_REMOVE(rseg->undo_list, undo);
 		trx_purge_add_undo_to_history(trx, undo, mtr);
@@ -1382,20 +1357,12 @@ trx_commit_in_memory(
 		ut_ad(rseg->trx_ref_count > 0);
 		--rseg->trx_ref_count;
 		mutex_exit(&rseg->mutex);
-
-		if (trx_undo_t*& insert = trx->rsegs.m_redo.old_insert) {
-			ut_ad(insert->rseg == rseg);
-			trx_undo_commit_cleanup(insert, false);
-			insert = NULL;
-		}
 	}
 
-	ut_ad(!trx->rsegs.m_redo.old_insert);
-
 	if (mtr != NULL) {
 		if (trx_undo_t*& undo = trx->rsegs.m_noredo.undo) {
 			ut_ad(undo->rseg == trx->rsegs.m_noredo.rseg);
-			trx_undo_commit_cleanup(undo, true);
+			trx_undo_commit_cleanup(undo);
 			undo = NULL;
 		}
 
@@ -1490,7 +1457,7 @@ void trx_commit_low(trx_t* trx, mtr_t* mtr)
 	ut_ad(!mtr || mtr->is_active());
 	ut_d(bool aborted = trx->in_rollback
 	     && trx->error_state == DB_DEADLOCK);
-	ut_ad(!mtr == (aborted || !trx->has_logged_or_recovered()));
+	ut_ad(!mtr == (aborted || !trx->has_logged()));
 	ut_ad(!mtr || !aborted);
 
 	/* undo_no is non-zero if we're doing the final commit. */
@@ -1577,10 +1544,7 @@ trx_commit(
 	mtr_t*	mtr;
 	mtr_t	local_mtr;
 
-	DBUG_EXECUTE_IF("ib_trx_commit_crash_before_trx_commit_start",
-			DBUG_SUICIDE(););
-
-	if (trx->has_logged_or_recovered()) {
+	if (trx->has_logged()) {
 		mtr = &local_mtr;
 		mtr->start();
 	} else {
@@ -1986,11 +1950,8 @@ trx_weight_ge(
 /** Prepare a transaction.
 @return	log sequence number that makes the XA PREPARE durable
 @retval	0	if no changes needed to be made durable */
-static
-lsn_t
-trx_prepare_low(trx_t* trx)
+static lsn_t trx_prepare_low(trx_t *trx)
 {
-	ut_ad(!trx->rsegs.m_redo.old_insert);
 	ut_ad(!trx->is_recovered);
 
 	mtr_t	mtr;
diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc
index 7254c830cde..dea85c40473 100644
--- a/storage/innobase/trx/trx0undo.cc
+++ b/storage/innobase/trx/trx0undo.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2020, MariaDB Corporation.
+Copyright (c) 2014, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -34,6 +34,7 @@ Created 3/26/1996 Heikki Tuuri
 #include "trx0purge.h"
 #include "trx0rec.h"
 #include "trx0rseg.h"
+#include "log.h"
 
 /* How should the old versions in the history list be managed?
    ----------------------------------------------------------
@@ -1011,13 +1012,8 @@ loop:
 }
 
 /** Frees an undo log segment which is not in the history list.
-@param[in]	undo	undo log
-@param[in]	noredo	whether the undo tablespace is redo logged */
-static
-void
-trx_undo_seg_free(
-	const trx_undo_t*	undo,
-	bool			noredo)
+@param undo	temporary undo log */
+static void trx_undo_seg_free(const trx_undo_t *undo)
 {
 	trx_rseg_t*	rseg;
 	fseg_header_t*	file_seg;
@@ -1029,16 +1025,12 @@ trx_undo_seg_free(
 	rseg = undo->rseg;
 
 	do {
-
-		mtr_start(&mtr);
-
-		if (noredo) {
-			mtr.set_log_mode(MTR_LOG_NO_REDO);
-		}
+		mtr.start();
+		mtr.set_log_mode(MTR_LOG_NO_REDO);
 
 		mutex_enter(&(rseg->mutex));
 
-		seg_header = trx_undo_page_get(page_id_t(undo->rseg->space->id,
+		seg_header = trx_undo_page_get(page_id_t(SRV_TMP_SPACE_ID,
 							 undo->hdr_page_no),
 					       &mtr)
 			+ TRX_UNDO_SEG_HDR;
@@ -1069,10 +1061,11 @@ trx_undo_seg_free(
 @param[in]	id		rollback segment slot
 @param[in]	page_no		undo log segment page number
 @param[in,out]	max_trx_id	the largest observed transaction ID
-@return	size of the undo log in pages */
-ulint
-trx_undo_mem_create_at_db_start(trx_rseg_t* rseg, ulint id, ulint page_no,
-				trx_id_t& max_trx_id)
+@return	the undo log
+@retval nullptr on error */
+trx_undo_t *
+trx_undo_mem_create_at_db_start(trx_rseg_t *rseg, ulint id, uint32_t page_no,
+                                trx_id_t &max_trx_id)
 {
 	mtr_t		mtr;
 	XID		xid;
@@ -1082,16 +1075,56 @@ trx_undo_mem_create_at_db_start(trx_rseg_t* rseg, ulint id, ulint page_no,
 	mtr.start();
 	const page_t* undo_page = trx_undo_page_get(
 		page_id_t(rseg->space->id, page_no), &mtr);
-	const ulint type = mach_read_from_2(
-		TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE + undo_page);
-	ut_ad(type == 0 || type == TRX_UNDO_INSERT || type == TRX_UNDO_UPDATE);
+	const uint16_t type = mach_read_from_2(TRX_UNDO_PAGE_HDR
+					       + TRX_UNDO_PAGE_TYPE
+					       + undo_page);
+	switch (type) {
+	case 0:
+	case 2: /* TRX_UNDO_UPDATE */
+		break;
+	case 1: /* TRX_UNDO_INSERT */
+		sql_print_error("InnoDB: upgrade from older version than"
+				" MariaDB 10.3 requires clean shutdown");
+		goto corrupted;
+	default:
+		sql_print_error("InnoDB: unsupported undo header type %u",
+				type);
+	corrupted:
+		mtr.commit();
+		return NULL;
+	}
 
-	uint state = mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_STATE
-				      + undo_page);
-	uint offset = mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_LAST_LOG
-				       + undo_page);
+	uint16_t offset = mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_LAST_LOG
+					   + undo_page);
+	if (offset < TRX_UNDO_SEG_HDR + TRX_UNDO_SEG_HDR_SIZE ||
+	    offset >= srv_page_size - TRX_UNDO_LOG_OLD_HDR_SIZE) {
+		sql_print_error("InnoDB: invalid undo header offset %u",
+				offset);
+		goto corrupted;
+	}
 
-	const trx_ulogf_t*	undo_header = undo_page + offset;
+	const trx_ulogf_t* const undo_header = undo_page + offset;
+	uint16_t state = mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_STATE
+					  + undo_page);
+	switch (state) {
+	case TRX_UNDO_ACTIVE:
+	case TRX_UNDO_PREPARED:
+		break;
+	default:
+		sql_print_error("InnoDB: unsupported undo header state %u",
+				state);
+		goto corrupted;
+	case TRX_UNDO_TO_PURGE:
+	case TRX_UNDO_CACHED:
+		trx_id_t id = mach_read_from_8(TRX_UNDO_TRX_NO + undo_header);
+		if (id >> 48) {
+			sql_print_error("InnoDB: corrupted TRX_NO %llx", id);
+			goto corrupted;
+		}
+		if (id > max_trx_id) {
+			max_trx_id = id;
+		}
+	}
 
 	/* Read X/Open XA transaction identification if it exists, or
 	set it to NULL. */
@@ -1103,6 +1136,10 @@ trx_undo_mem_create_at_db_start(trx_rseg_t* rseg, ulint id, ulint page_no,
 	}
 
 	trx_id_t trx_id = mach_read_from_8(undo_header + TRX_UNDO_TRX_ID);
+	if (trx_id >> 48) {
+		sql_print_error("InnoDB: corrupted TRX_ID %llx", trx_id);
+		goto corrupted;
+	}
 	if (trx_id > max_trx_id) {
 		max_trx_id = trx_id;
 	}
@@ -1111,61 +1148,45 @@ trx_undo_mem_create_at_db_start(trx_rseg_t* rseg, ulint id, ulint page_no,
 	trx_undo_t* undo = trx_undo_mem_create(
 		rseg, id, trx_id, &xid, page_no, offset);
 	mutex_exit(&rseg->mutex);
+	if (!undo) {
+		return undo;
+	}
 
 	undo->dict_operation = undo_header[TRX_UNDO_DICT_TRANS];
 	undo->table_id = mach_read_from_8(undo_header + TRX_UNDO_TABLE_ID);
 	undo->size = flst_get_len(TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST
 				  + undo_page);
 
-	if (UNIV_UNLIKELY(state == TRX_UNDO_TO_FREE)) {
-		/* This is an old-format insert_undo log segment that
-		is being freed. The page list is inconsistent. */
-		ut_ad(type == TRX_UNDO_INSERT);
-		state = TRX_UNDO_TO_PURGE;
+	fil_addr_t	last_addr = flst_get_last(
+		TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + undo_page, &mtr);
+
+	undo->last_page_no = last_addr.page;
+	undo->top_page_no = last_addr.page;
+
+	page_t* last_page = trx_undo_page_get(
+		page_id_t(rseg->space->id, undo->last_page_no), &mtr);
+
+	if (const trx_undo_rec_t* rec = trx_undo_page_get_last_rec(
+		    last_page, page_no, offset)) {
+		undo->top_offset = ulint(rec - last_page);
+		undo->top_undo_no = trx_undo_rec_get_undo_no(rec);
+		ut_ad(!undo->empty());
 	} else {
-		if (state == TRX_UNDO_TO_PURGE
-		    || state == TRX_UNDO_CACHED) {
-			trx_id_t id = mach_read_from_8(TRX_UNDO_TRX_NO
-						       + undo_header);
-			if (id > max_trx_id) {
-				max_trx_id = id;
-			}
-		}
-
-		fil_addr_t	last_addr = flst_get_last(
-			TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + undo_page,
-			&mtr);
-
-		undo->last_page_no = last_addr.page;
-		undo->top_page_no = last_addr.page;
-
-		page_t* last_page = trx_undo_page_get(
-			page_id_t(rseg->space->id, undo->last_page_no), &mtr);
-
-		if (const trx_undo_rec_t* rec = trx_undo_page_get_last_rec(
-			    last_page, page_no, offset)) {
-			undo->top_offset = ulint(rec - last_page);
-			undo->top_undo_no = trx_undo_rec_get_undo_no(rec);
-			ut_ad(!undo->empty());
-		} else {
-			undo->top_undo_no = IB_ID_MAX;
-			ut_ad(undo->empty());
-		}
+		undo->top_undo_no = IB_ID_MAX;
+		ut_ad(undo->empty());
 	}
 
 	undo->state = state;
 
 	if (state != TRX_UNDO_CACHED) {
-		UT_LIST_ADD_LAST(type == TRX_UNDO_INSERT
-				 ? rseg->old_insert_list
-				 : rseg->undo_list, undo);
+		UT_LIST_ADD_LAST(rseg->undo_list, undo);
 	} else {
 		UT_LIST_ADD_LAST(rseg->undo_cached, undo);
 		MONITOR_INC(MONITOR_NUM_UNDO_SLOT_CACHED);
 	}
 
 	mtr.commit();
-	return undo->size;
+	return undo;
 }
 
 /********************************************************************//**
@@ -1577,22 +1598,18 @@ trx_undo_set_state_at_prepare(
 	return(undo_page);
 }
 
-/** Free an old insert or temporary undo log after commit or rollback.
+/** Free temporary undo log after commit or rollback.
 The information is not needed after a commit or rollback, therefore
 the data can be discarded.
-@param[in,out]	undo	undo log
-@param[in]	is_temp	whether this is temporary undo log */
-void
-trx_undo_commit_cleanup(trx_undo_t* undo, bool is_temp)
+@param undo     temporary undo log */
+void trx_undo_commit_cleanup(trx_undo_t *undo)
 {
 	trx_rseg_t*	rseg	= undo->rseg;
-	ut_ad(is_temp == !rseg->is_persistent());
-	ut_ad(!is_temp || 0 == UT_LIST_GET_LEN(rseg->old_insert_list));
+	ut_ad(rseg->space == fil_system.temp_space);
 
 	mutex_enter(&rseg->mutex);
 
-	UT_LIST_REMOVE(is_temp ? rseg->undo_list : rseg->old_insert_list,
-		       undo);
+	UT_LIST_REMOVE(rseg->undo_list, undo);
 
 	if (undo->state == TRX_UNDO_CACHED) {
 		UT_LIST_ADD_FIRST(rseg->undo_cached, undo);
@@ -1602,7 +1619,7 @@ trx_undo_commit_cleanup(trx_undo_t* undo, bool is_temp)
 
 		/* Delete first the undo log segment in the file */
 		mutex_exit(&rseg->mutex);
-		trx_undo_seg_free(undo, is_temp);
+		trx_undo_seg_free(undo);
 		mutex_enter(&rseg->mutex);
 
 		ut_ad(rseg->curr_size > undo->size);
@@ -1615,15 +1632,13 @@ trx_undo_commit_cleanup(trx_undo_t* undo, bool is_temp)
 }
 
 /** At shutdown, frees the undo logs of a transaction. */
-void
-trx_undo_free_at_shutdown(trx_t *trx)
+void trx_undo_free_at_shutdown(trx_t *trx)
 {
 	if (trx_undo_t*& undo = trx->rsegs.m_redo.undo) {
 		switch (undo->state) {
 		case TRX_UNDO_PREPARED:
 			break;
 		case TRX_UNDO_CACHED:
-		case TRX_UNDO_TO_FREE:
 		case TRX_UNDO_TO_PURGE:
 			ut_ad(trx_state_eq(trx,
 					   TRX_STATE_COMMITTED_IN_MEMORY));
@@ -1644,34 +1659,6 @@ trx_undo_free_at_shutdown(trx_t *trx)
 		ut_free(undo);
 		undo = NULL;
 	}
-
-	if (trx_undo_t*& undo = trx->rsegs.m_redo.old_insert) {
-		switch (undo->state) {
-		case TRX_UNDO_PREPARED:
-			break;
-		case TRX_UNDO_CACHED:
-		case TRX_UNDO_TO_FREE:
-		case TRX_UNDO_TO_PURGE:
-			ut_ad(trx_state_eq(trx,
-					   TRX_STATE_COMMITTED_IN_MEMORY));
-			/* fall through */
-		case TRX_UNDO_ACTIVE:
-			/* trx_t::commit_state() assigns
-			trx->state = TRX_STATE_COMMITTED_IN_MEMORY. */
-			ut_a(!srv_was_started
-			     || srv_read_only_mode
-			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO
-			     || srv_fast_shutdown);
-			break;
-		default:
-			ut_error;
-		}
-
-		UT_LIST_REMOVE(trx->rsegs.m_redo.rseg->old_insert_list, undo);
-		ut_free(undo);
-		undo = NULL;
-	}
-
 	if (trx_undo_t*& undo = trx->rsegs.m_noredo.undo) {
 		ut_a(undo->state == TRX_UNDO_PREPARED);