From 696de6d06c0eeaf7b20d5f89278ed7d62a9f204f Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 13 Sep 2021 12:37:07 +0530 Subject: [PATCH 01/11] MDEV-25702 Auxiliary FTS table evicts during optimize table InnoDB could evict the fts auxiliary table in row_fts_merge_insert(). So bulk insert could be dealing with garbage FTS auxiliary table.Patch should delay closing the table in row_fts_merge_insert(). --- storage/innobase/row/row0ftsort.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 29e126290cf..c274c63182e 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -1664,7 +1664,6 @@ row_fts_merge_insert( aux_table = dict_table_open_on_name(aux_table_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); ut_ad(aux_table != NULL); - dict_table_close(aux_table, FALSE, FALSE); aux_index = dict_table_get_first_index(aux_table); /* Create bulk load instance */ @@ -1789,6 +1788,8 @@ row_fts_merge_insert( } exit: + dict_table_close(aux_table, FALSE, FALSE); + fts_sql_commit(trx); trx->op_info = ""; From 8937762ead9623f4bcd23d445319c08e07d6c321 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 15 Sep 2021 14:19:24 +0200 Subject: [PATCH 02/11] MDEV-26573 : A static analyzer warning about ds_archive.cc This file had not been compiled for long time. Remove this from the tree. --- extra/mariabackup/datasink.cc | 8 - extra/mariabackup/datasink.h | 1 - extra/mariabackup/ds_archive.cc | 281 -------------------------------- extra/mariabackup/ds_archive.h | 28 ---- 4 files changed, 318 deletions(-) delete mode 100644 extra/mariabackup/ds_archive.cc delete mode 100644 extra/mariabackup/ds_archive.h diff --git a/extra/mariabackup/datasink.cc b/extra/mariabackup/datasink.cc index 4235a46ea00..ecd60a7df95 100644 --- a/extra/mariabackup/datasink.cc +++ b/extra/mariabackup/datasink.cc @@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include "common.h" #include "datasink.h" #include "ds_compress.h" -#include "ds_archive.h" #include "ds_xbstream.h" #include "ds_local.h" #include "ds_stdout.h" @@ -44,13 +43,6 @@ ds_create(const char *root, ds_type_t type) case DS_TYPE_LOCAL: ds = &datasink_local; break; - case DS_TYPE_ARCHIVE: -#ifdef HAVE_LIBARCHIVE - ds = &datasink_archive; -#else - die("mariabackup was built without libarchive support"); -#endif - break; case DS_TYPE_XBSTREAM: ds = &datasink_xbstream; break; diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index 5c82556b9ba..4bede4ec9e7 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -63,7 +63,6 @@ static inline int dummy_remove(const char *) { typedef enum { DS_TYPE_STDOUT, DS_TYPE_LOCAL, - DS_TYPE_ARCHIVE, DS_TYPE_XBSTREAM, DS_TYPE_COMPRESS, DS_TYPE_ENCRYPT, diff --git a/extra/mariabackup/ds_archive.cc b/extra/mariabackup/ds_archive.cc deleted file mode 100644 index 18f13fbcf3a..00000000000 --- a/extra/mariabackup/ds_archive.cc +++ /dev/null @@ -1,281 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -Streaming implementation for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -#include -#include -#include -#include "common.h" -#include "datasink.h" - -#if ARCHIVE_VERSION_NUMBER < 3000000 -#define archive_write_add_filter_none(X) archive_write_set_compression_none(X) -#define archive_write_free(X) archive_write_finish(X) -#endif - -typedef struct { - struct archive *archive; - ds_file_t *dest_file; - pthread_mutex_t mutex; -} ds_archive_ctxt_t; - -typedef struct { - struct archive_entry *entry; - ds_archive_ctxt_t *archive_ctxt; -} ds_archive_file_t; - - -/*********************************************************************** -General archive interface */ - -static ds_ctxt_t *archive_init(const char *root); -static ds_file_t *archive_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); -static int archive_write(ds_file_t *file, const void *buf, size_t len); -static int archive_close(ds_file_t *file); -static void archive_deinit(ds_ctxt_t *ctxt); - -datasink_t datasink_archive = { - &archive_init, - &archive_open, - &archive_write, - &archive_close, - &dummy_remove, - &archive_deinit -}; - -static -int -my_archive_open_callback(struct archive *a __attribute__((unused)), - void *data __attribute__((unused))) -{ - return ARCHIVE_OK; -} - -static -ssize_t -my_archive_write_callback(struct archive *a __attribute__((unused)), - void *data, const void *buffer, size_t length) -{ - ds_archive_ctxt_t *archive_ctxt; - - archive_ctxt = (ds_archive_ctxt_t *) data; - - xb_ad(archive_ctxt != NULL); - xb_ad(archive_ctxt->dest_file != NULL); - - if (!ds_write(archive_ctxt->dest_file, buffer, length)) { - return length; - } - return -1; -} - -static -int -my_archive_close_callback(struct archive *a __attribute__((unused)), - void *data __attribute__((unused))) -{ - return ARCHIVE_OK; -} - -static -ds_ctxt_t * -archive_init(const char *root __attribute__((unused))) -{ - ds_ctxt_t *ctxt; - ds_archive_ctxt_t *archive_ctxt; - struct archive *a; - - ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_archive_ctxt_t), - MYF(MY_FAE)); - archive_ctxt = (ds_archive_ctxt_t *)(ctxt + 1); - - if (pthread_mutex_init(&archive_ctxt->mutex, NULL)) { - msg("archive_init: pthread_mutex_init() failed.\n"); - goto err; - } - - a = archive_write_new(); - if (a == NULL) { - msg("archive_write_new() failed.\n"); - goto err; - } - - archive_ctxt->archive = a; - archive_ctxt->dest_file = NULL; - - if(archive_write_add_filter_none(a) != ARCHIVE_OK || - archive_write_set_format_pax_restricted(a) != ARCHIVE_OK || - /* disable internal buffering so we don't have to flush the - output in xtrabackup */ - archive_write_set_bytes_per_block(a, 0) != ARCHIVE_OK) { - msg("failed to set libarchive archive options: %s\n", - archive_error_string(a)); - archive_write_free(a); - goto err; - } - - if (archive_write_open(a, archive_ctxt, my_archive_open_callback, - my_archive_write_callback, - my_archive_close_callback) != ARCHIVE_OK) { - msg("cannot open output archive.\n"); - return NULL; - } - - ctxt->ptr = archive_ctxt; - - return ctxt; - -err: - my_free(ctxt); - return NULL; -} - -static -ds_file_t * -archive_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) -{ - ds_archive_ctxt_t *archive_ctxt; - ds_ctxt_t *dest_ctxt; - ds_file_t *file; - ds_archive_file_t *archive_file; - - struct archive *a; - struct archive_entry *entry; - - xb_ad(ctxt->pipe_ctxt != NULL); - dest_ctxt = ctxt->pipe_ctxt; - - archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr; - - pthread_mutex_lock(&archive_ctxt->mutex); - if (archive_ctxt->dest_file == NULL) { - archive_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); - if (archive_ctxt->dest_file == NULL) { - return NULL; - } - } - pthread_mutex_unlock(&archive_ctxt->mutex); - - file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + - sizeof(ds_archive_file_t), - MYF(MY_FAE)); - - archive_file = (ds_archive_file_t *) (file + 1); - - a = archive_ctxt->archive; - - entry = archive_entry_new(); - if (entry == NULL) { - msg("archive_entry_new() failed.\n"); - goto err; - } - - archive_entry_set_size(entry, mystat->st_size); - archive_entry_set_mode(entry, 0660); - archive_entry_set_filetype(entry, AE_IFREG); - archive_entry_set_pathname(entry, path); - archive_entry_set_mtime(entry, mystat->st_mtime, 0); - - archive_file->entry = entry; - archive_file->archive_ctxt = archive_ctxt; - - if (archive_write_header(a, entry) != ARCHIVE_OK) { - msg("archive_write_header() failed.\n"); - archive_entry_free(entry); - goto err; - } - - file->ptr = archive_file; - file->path = archive_ctxt->dest_file->path; - - return file; - -err: - if (archive_ctxt->dest_file) { - ds_close(archive_ctxt->dest_file); - archive_ctxt->dest_file = NULL; - } - my_free(file); - - return NULL; -} - -static -int -archive_write(ds_file_t *file, const void *buf, size_t len) -{ - ds_archive_file_t *archive_file; - struct archive *a; - - archive_file = (ds_archive_file_t *) file->ptr; - - a = archive_file->archive_ctxt->archive; - - xb_ad(archive_file->archive_ctxt->dest_file != NULL); - if (archive_write_data(a, buf, len) < 0) { - msg("archive_write_data() failed: %s (errno = %d)\n", - archive_error_string(a), archive_errno(a)); - return 1; - } - - return 0; -} - -static -int -archive_close(ds_file_t *file) -{ - ds_archive_file_t *archive_file; - int rc = 0; - - archive_file = (ds_archive_file_t *)file->ptr; - - archive_entry_free(archive_file->entry); - - my_free(file); - - return rc; -} - -static -void -archive_deinit(ds_ctxt_t *ctxt) -{ - struct archive *a; - ds_archive_ctxt_t *archive_ctxt; - - archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr; - - a = archive_ctxt->archive; - - if (archive_write_close(a) != ARCHIVE_OK) { - msg("archive_write_close() failed.\n"); - } - archive_write_free(a); - - if (archive_ctxt->dest_file) { - ds_close(archive_ctxt->dest_file); - archive_ctxt->dest_file = NULL; - } - - pthread_mutex_destroy(&archive_ctxt->mutex); - - my_free(ctxt); -} diff --git a/extra/mariabackup/ds_archive.h b/extra/mariabackup/ds_archive.h deleted file mode 100644 index f419fca0c9f..00000000000 --- a/extra/mariabackup/ds_archive.h +++ /dev/null @@ -1,28 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -Streaming interface for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -#ifndef DS_ARCHIVE_H -#define DS_ARCHIVE_H - -#include "datasink.h" - -extern datasink_t datasink_archive; - -#endif From b1351c15946349f9daa7e5297fb2ac6f3139e4a8 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 15 Sep 2021 14:55:45 +0200 Subject: [PATCH 03/11] MDEV-26574 An improper locking bug due to unreleased lock in the ds_xbstream.cc release lock in all as cases n xbstream_open, also fix the case where malloc would return NULL. --- extra/mariabackup/ds_xbstream.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/extra/mariabackup/ds_xbstream.cc b/extra/mariabackup/ds_xbstream.cc index 3b60456f8ed..798d7da6da4 100644 --- a/extra/mariabackup/ds_xbstream.cc +++ b/extra/mariabackup/ds_xbstream.cc @@ -126,15 +126,19 @@ xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) pthread_mutex_lock(&stream_ctxt->mutex); if (stream_ctxt->dest_file == NULL) { stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); - if (stream_ctxt->dest_file == NULL) { - return NULL; - } } pthread_mutex_unlock(&stream_ctxt->mutex); + if (stream_ctxt->dest_file == NULL) { + return NULL; + } file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + sizeof(ds_stream_file_t), MYF(MY_FAE)); + if (!file) { + msg("my_malloc() failed."); + goto err; + } stream_file = (ds_stream_file_t *) (file + 1); xbstream = stream_ctxt->xbstream; From dabc3329c30d3be53449d49f501acaf0b0788dbf Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 16 Sep 2021 20:42:26 +1000 Subject: [PATCH 04/11] MDEV-26622: man mysqldump - insert-ignore not insert-into Thanks Karl Levik --- man/mysqldump.1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/man/mysqldump.1 b/man/mysqldump.1 index 2cfe308836a..907eeda4d38 100644 --- a/man/mysqldump.1 +++ b/man/mysqldump.1 @@ -2261,7 +2261,7 @@ servers \- remote (federated) servers as \fBCREATE SERVER\fR\&. .sp -1 .IP \(bu 2.3 .\} -stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS), are dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-into\fR is specified) statements without (re)creating tables\&. +stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS), are dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-ignore\fR is specified) statements without (re)creating tables\&. .RE .RS 4 .ie n \{\ @@ -2271,17 +2271,17 @@ stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS .sp -1 .IP \(bu 2.3 .\} -timezones \- timezone related system tables dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-into\fR is specified) statements without (re)creating tables\&. +timezones \- timezone related system tables dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-ignore\fR is specified) statements without (re)creating tables\&. .RE .sp -The format of the output is affected by \fB\-\-replace\fR and \fB\-\-insert\-into\fR\&. The \fB\-\-replace\fR option will output \fBCREATE OR REPLACE\fR +The format of the output is affected by \fB\-\-replace\fR and \fB\-\-insert\-ignore\fR\&. The \fB\-\-replace\fR option will output \fBCREATE OR REPLACE\fR forms of SQL, and also \fBDROP IF EXISTS\fR prior to \fBCREATE\fR, if a \fBCREATE OR REPLACE\fR option isn't available. .sp With \fB\-\-system=user\fR (or \fBall\fR), and \fB\-\-replace\fR, SQL is generated to generate an error if attempting to import the dump with a connection user that is being replaced within the dump\&. .sp -The \fB\-\-insert\-into\fR option will cause \fBCREATE IF NOT EXIST\fR forms of SQL to generated if available. +The \fB\-\-insert\-ignore\fR option will cause \fBCREATE IF NOT EXIST\fR forms of SQL to generated if available. .sp -For stats, and timezones, \fB\-\-replace\fR and \fB\-\-insert\-into\fR have the usual effects. +For stats, and timezones, \fB\-\-replace\fR and \fB\-\-insert\-ignore\fR have the usual effects. .sp Enabling specific options here will cause the relevant tables in the mysql database to be ignored when dumping the mysql database or \fB\-\-all\-databases\fR\&. .sp From dce490e9d4d91a7e00111bd515db558cadc19d06 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 15 Sep 2021 16:37:54 +0200 Subject: [PATCH 05/11] Give less memory to get reliable error. --- mysql-test/r/truncate_notembedded.result | 4 ++-- mysql-test/t/truncate_notembedded.test | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/truncate_notembedded.result b/mysql-test/r/truncate_notembedded.result index 18a01c684e7..dabd5474141 100644 --- a/mysql-test/r/truncate_notembedded.result +++ b/mysql-test/r/truncate_notembedded.result @@ -5,7 +5,7 @@ CREATE TABLE t1 (a INT) ENGINE=MyISAM; LOCK TABLE t1 READ; connect con1,localhost,root,,test; -SET SESSION max_session_mem_used= 65536; +SET SESSION max_session_mem_used= 45500; LOCK TABLE t1 WRITE; connection default; SELECT * FROM t1; @@ -13,7 +13,7 @@ a UNLOCK TABLES; connection con1; TRUNCATE TABLE t1; -ERROR HY000: The MariaDB server is running with the --max-thread-mem-used=65536 option so it cannot execute this statement +ERROR HY000: The MariaDB server is running with the --max-thread-mem-used=45500 option so it cannot execute this statement disconnect con1; connection default; DROP TABLE t1; diff --git a/mysql-test/t/truncate_notembedded.test b/mysql-test/t/truncate_notembedded.test index 7e186c8f7d5..3addad2d975 100644 --- a/mysql-test/t/truncate_notembedded.test +++ b/mysql-test/t/truncate_notembedded.test @@ -9,7 +9,7 @@ CREATE TABLE t1 (a INT) ENGINE=MyISAM; LOCK TABLE t1 READ; --connect (con1,localhost,root,,test) -SET SESSION max_session_mem_used= 65536; +SET SESSION max_session_mem_used= 45500; --send LOCK TABLE t1 WRITE; From 496d3dded4559ef8cb64ee4fc68eb4d461d78bd5 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 17 Sep 2021 19:37:29 +0530 Subject: [PATCH 06/11] MDEV-15675 encryption.innodb_encryption failed in buildbot with timeout Test case fail to include undo tablespace while waiting for the encryption thread to encrypt all existing tablespace --- mysql-test/suite/encryption/r/innodb_encryption.result | 2 +- mysql-test/suite/encryption/t/innodb_encryption.test | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encryption.result b/mysql-test/suite/encryption/r/innodb_encryption.result index 75614c6b88c..22b4c3a5ad6 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption.result +++ b/mysql-test/suite/encryption/r/innodb_encryption.result @@ -19,7 +19,7 @@ innodb_system # Success! # Now turn off encryption and wait for threads to decrypt everything SET GLOBAL innodb_encrypt_tables = off; -# Wait max 10 min for key encryption threads to encrypt all spaces +# Wait max 10 min for key encryption threads to decrypt all spaces SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME NOT LIKE 'innodb_undo%' AND NAME NOT LIKE 'mysql/innodb_%_stats'; NAME diff --git a/mysql-test/suite/encryption/t/innodb_encryption.test b/mysql-test/suite/encryption/t/innodb_encryption.test index cd844af0649..303e74fdba4 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption.test +++ b/mysql-test/suite/encryption/t/innodb_encryption.test @@ -14,7 +14,9 @@ SHOW VARIABLES LIKE 'innodb_encrypt%'; SET GLOBAL innodb_encrypt_tables = ON; ---let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'` +let $undo_count= `select @@global.innodb_undo_tablespaces`; + +--let $tables_count= `select count(*) + 1 + $undo_count from information_schema.tables where engine = 'InnoDB'` --echo # Wait max 10 min for key encryption threads to encrypt all spaces --let $wait_timeout= 600 @@ -33,7 +35,7 @@ AND NAME NOT LIKE 'innodb_undo%' AND NAME NOT LIKE 'mysql/innodb_%_stats'; --echo # Now turn off encryption and wait for threads to decrypt everything SET GLOBAL innodb_encrypt_tables = off; ---echo # Wait max 10 min for key encryption threads to encrypt all spaces +--echo # Wait max 10 min for key encryption threads to decrypt all spaces --let $wait_timeout= 600 --let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; --source include/wait_condition.inc From 3209bc667fd3aeb7b3c3709ca927c2e77a5bf273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 18 Sep 2021 15:47:52 +0300 Subject: [PATCH 07/11] MDEV-26636: InnoDB defragmentation statistics cause races on TEMPORARY TABLE btr_defragment_save_defrag_stats_if_needed(): Do not save defragmentation statistics for temporary tables. They are exempt of defragmentation anyway (ha_innobase::optimize() never invokes defragmentation for them), and the user-visible names are not available inside InnoDB. Furthermore, InnoDB assumes that temporary tables are never accessed by other threads than the one that handles the session with which the temporary table is associated with. Furthermore, we simplify the test innodb.innodb_defrag_stats and include a test case that demonstrates that defragmentation statistics are no longer being saved for temporary tables. --- .../suite/innodb/r/innodb_defrag_stats.result | 103 ++++++------------ .../suite/innodb/t/innodb_defrag_stats.test | 72 ++++-------- storage/innobase/btr/btr0defragment.cc | 3 +- 3 files changed, 62 insertions(+), 116 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_defrag_stats.result b/mysql-test/suite/innodb/r/innodb_defrag_stats.result index 7092688f07b..6c5fe1817e2 100644 --- a/mysql-test/suite/innodb/r/innodb_defrag_stats.result +++ b/mysql-test/suite/innodb/r/innodb_defrag_stats.result @@ -1,22 +1,8 @@ -DROP TABLE if exists t1; -select @@global.innodb_stats_persistent; -@@global.innodb_stats_persistent -0 -set global innodb_defragment_stats_accuracy = 20; +SET GLOBAL innodb_defragment_stats_accuracy = 20; +DELETE FROM mysql.innodb_index_stats; # Create table. CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB; -# Populate data -INSERT INTO t1 VALUES(1, REPEAT('A', 256)); -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; +INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1_to_1024; # Not enough page splits to trigger persistent stats write yet. select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) = 0 @@ -27,7 +13,7 @@ count(stat_value) = 0 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); count(stat_value) = 0 1 -INSERT INTO t1 (b) SELECT b from t1; +INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1025_to_2048; # Persistent stats recorded. select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) > 0 @@ -39,6 +25,7 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like count(stat_value) > 0 1 # Delete some rows. +BEGIN; delete from t1 where a between 100 * 20 and 100 * 20 + 30; delete from t1 where a between 100 * 19 and 100 * 19 + 30; delete from t1 where a between 100 * 18 and 100 * 18 + 30; @@ -59,8 +46,7 @@ delete from t1 where a between 100 * 4 and 100 * 4 + 30; delete from t1 where a between 100 * 3 and 100 * 3 + 30; delete from t1 where a between 100 * 2 and 100 * 2 + 30; delete from t1 where a between 100 * 1 and 100 * 1 + 30; -# Server Restarted -# Confirm persistent stats still there after restart. +COMMIT; select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) > 0 1 @@ -73,9 +59,6 @@ count(stat_value) > 0 optimize table t1; Table Op Msg_type Msg_text test.t1 optimize status OK -select sleep(2); -sleep(2) -0 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) > 0 1 @@ -108,9 +91,6 @@ count(stat_value) > 0 1 # Table rename should cause stats rename. rename table t1 to t2; -select sleep(1); -sleep(1) -0 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) = 0 1 @@ -129,47 +109,36 @@ count(stat_value) > 0 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); count(stat_value) > 0 1 -# Drop index should cause stats drop. +# Drop index should cause stats drop, but will not. drop index SECOND on t2; -select sleep(3); -sleep(3) -0 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_page_split'); -count(stat_value) > 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_pages_freed'); -count(stat_value) > 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_leaf_pages_defrag'); -count(stat_value) > 0 -1 -Server Restarted -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); -count(stat_value) = 0 -1 -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); -count(stat_value) = 0 -1 -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); -count(stat_value) = 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_page_split'); -count(stat_value) > 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); -count(stat_value) > 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); -count(stat_value) > 0 -1 +SELECT stat_name, stat_value>0 FROM mysql.innodb_index_stats +WHERE table_name like '%t2%' AND index_name='SECOND'; +stat_name stat_value>0 +n_leaf_pages_defrag 1 +n_leaf_pages_reserved 1 +n_page_split 1 +n_pages_freed 1 +# +# MDEV-26636: Statistics must not be written for temporary tables +# +SET GLOBAL innodb_defragment_stats_accuracy = 1; +CREATE TEMPORARY TABLE t (a INT PRIMARY KEY, c CHAR(255) NOT NULL) +ENGINE=InnoDB; +INSERT INTO t SELECT seq, '' FROM seq_1_to_100; +SELECT * FROM mysql.innodb_index_stats where table_name like '%t1%'; +database_name table_name index_name last_update stat_name stat_value sample_size stat_description +SELECT table_name, index_name, stat_name, stat_value>0 +FROM mysql.innodb_index_stats; +table_name index_name stat_name stat_value>0 +t2 PRIMARY n_leaf_pages_defrag 1 +t2 PRIMARY n_leaf_pages_reserved 1 +t2 PRIMARY n_page_split 1 +t2 PRIMARY n_pages_freed 1 +t2 SECOND n_leaf_pages_defrag 1 +t2 SECOND n_leaf_pages_reserved 1 +t2 SECOND n_page_split 1 +t2 SECOND n_pages_freed 1 # Clean up DROP TABLE t2; -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_page_split'); -count(stat_value) = 0 -1 -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); -count(stat_value) = 0 -1 -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); -count(stat_value) = 0 -1 +SELECT * FROM mysql.innodb_index_stats; +database_name table_name index_name last_update stat_name stat_value sample_size stat_description diff --git a/mysql-test/suite/innodb/t/innodb_defrag_stats.test b/mysql-test/suite/innodb/t/innodb_defrag_stats.test index 2a5026a68e5..e1e88a07477 100644 --- a/mysql-test/suite/innodb/t/innodb_defrag_stats.test +++ b/mysql-test/suite/innodb/t/innodb_defrag_stats.test @@ -1,41 +1,23 @@ --source include/have_innodb.inc ---source include/big_test.inc --source include/not_valgrind.inc --source include/not_embedded.inc +--source include/have_sequence.inc ---disable_warnings -DROP TABLE if exists t1; ---enable_warnings +SET GLOBAL innodb_defragment_stats_accuracy = 20; ---disable_query_log -let $innodb_defragment_stats_accuracy_orig=`select @@innodb_defragment_stats_accuracy`; ---enable_query_log - -select @@global.innodb_stats_persistent; -set global innodb_defragment_stats_accuracy = 20; +DELETE FROM mysql.innodb_index_stats; --echo # Create table. CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB; ---echo # Populate data -INSERT INTO t1 VALUES(1, REPEAT('A', 256)); -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; +INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1_to_1024; --echo # Not enough page splits to trigger persistent stats write yet. select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); -INSERT INTO t1 (b) SELECT b from t1; +INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1025_to_2048; --echo # Persistent stats recorded. select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); @@ -43,6 +25,7 @@ select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); --echo # Delete some rows. +BEGIN; let $num_delete = 20; while ($num_delete) { @@ -50,17 +33,13 @@ while ($num_delete) eval delete from t1 where a between $j and $j + 30; dec $num_delete; } +COMMIT; ---source include/restart_mysqld.inc ---echo # Server Restarted - ---echo # Confirm persistent stats still there after restart. select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); optimize table t1; -select sleep(2); select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); @@ -84,7 +63,6 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like --echo # Table rename should cause stats rename. rename table t1 to t2; -select sleep(1); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); @@ -94,32 +72,30 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); ---echo # Drop index should cause stats drop. +--echo # Drop index should cause stats drop, but will not. drop index SECOND on t2; -select sleep(3); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_page_split'); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_pages_freed'); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_leaf_pages_defrag'); +--sorted_result +SELECT stat_name, stat_value>0 FROM mysql.innodb_index_stats +WHERE table_name like '%t2%' AND index_name='SECOND'; + +--echo # +--echo # MDEV-26636: Statistics must not be written for temporary tables +--echo # +SET GLOBAL innodb_defragment_stats_accuracy = 1; +CREATE TEMPORARY TABLE t (a INT PRIMARY KEY, c CHAR(255) NOT NULL) +ENGINE=InnoDB; +INSERT INTO t SELECT seq, '' FROM seq_1_to_100; --source include/restart_mysqld.inc ---echo Server Restarted -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); +SELECT * FROM mysql.innodb_index_stats where table_name like '%t1%'; -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_page_split'); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); +--sorted_result +SELECT table_name, index_name, stat_name, stat_value>0 +FROM mysql.innodb_index_stats; --echo # Clean up DROP TABLE t2; -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_page_split'); -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); - ---disable_query_log -EVAL SET GLOBAL innodb_defragment_stats_accuracy = $innodb_defragment_stats_accuracy_orig; ---enable_query_log +SELECT * FROM mysql.innodb_index_stats; diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 38ef1db9cec..36dd9ab8470 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2012, 2014 Facebook, Inc. All Rights Reserved. -Copyright (C) 2014, 2019, MariaDB Corporation. +Copyright (C) 2014, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -312,6 +312,7 @@ btr_defragment_save_defrag_stats_if_needed( { if (srv_defragment_stats_accuracy != 0 // stats tracking disabled && dict_index_get_space(index) != 0 // do not track system tables + && !index->table->is_temporary() && index->stat_defrag_modified_counter >= srv_defragment_stats_accuracy) { dict_stats_defrag_pool_add(index); From f4d6d01782ad4c885f1a4041514ba58f7b7d4ad2 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 20 Sep 2021 12:10:53 +0200 Subject: [PATCH 08/11] MDEV-26441: Linux-dependent construct in SST scripts SST scripts currently use Linux-specific construction to create a temporary directory if the path prefix for that directory is specified by the user. This does not work with FreeBSD. This commit adds support for FreeBSD. No separate test required. --- scripts/wsrep_sst_mariabackup.sh | 4 +++- scripts/wsrep_sst_rsync.sh | 4 +++- scripts/wsrep_sst_xtrabackup-v2.sh | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 562b9b929f2..54632e5f79b 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -949,8 +949,10 @@ then tmpdir=$(parse_cnf "$encgroups" 'tmpdir') if [ -z "$tmpdir" ]; then xtmpdir="$(mktemp -d)" - else + elif [ "$OS" = 'Linux' ]; then xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir") + else + xtmpdir=$(TMPDIR="$tmpdir"; mktemp '-d') fi wsrep_log_info "Using '$xtmpdir' as mariabackup temporary directory" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index d90e87b68f2..e16ed75cb16 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -725,8 +725,10 @@ EOF tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then tmpfile="$(mktemp)" - else + elif [ "$OS" = 'Linux' ]; then tmpfile=$(mktemp "--tmpdir=$tmpdir") + else + tmpfile=$(TMPDIR="$tmpdir"; mktemp '-d') fi wsrep_log_info "Extracting binlog files:" diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 13a4a1d25c1..999c6c649ba 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -973,8 +973,10 @@ then tmpdir=$(parse_cnf "$encgroups" 'tmpdir') if [ -z "$tmpdir" ]; then xtmpdir="$(mktemp -d)" - else + elif [ "$OS" = 'Linux' ]; then xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir") + else + xtmpdir=$(TMPDIR="$tmpdir"; mktemp '-d') fi wsrep_log_info "Using '$xtmpdir' as xtrabackup temporary directory" From ac1c6738f973498e2d136d5e419b961c151aa318 Mon Sep 17 00:00:00 2001 From: Daniel Ye <35718816+GreaTdANie1@users.noreply.github.com> Date: Wed, 22 Sep 2021 17:54:22 +0800 Subject: [PATCH 09/11] MDEV-26545 Spider does not correctly handle UDF and stored function in where conds - Handle stored function conditions correctly, with the same logic as with UDFs. - When running queries on Spider SE, by default, we do not push down WHERE conditions containing usage of UDFs/stored functions to remote data nodes, unless the user demands (by setting spider_use_pushdown_udf). --- .../mysql-test/spider/r/udf_pushdown.result | 222 ++++++++++++++++++ .../mysql-test/spider/t/udf_pushdown.inc | 48 ++++ .../mysql-test/spider/t/udf_pushdown.test | 141 +++++++++++ storage/spider/spd_db_mysql.cc | 6 + storage/spider/spd_param.cc | 2 +- 5 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 storage/spider/mysql-test/spider/r/udf_pushdown.result create mode 100644 storage/spider/mysql-test/spider/t/udf_pushdown.inc create mode 100644 storage/spider/mysql-test/spider/t/udf_pushdown.test diff --git a/storage/spider/mysql-test/spider/r/udf_pushdown.result b/storage/spider/mysql-test/spider/r/udf_pushdown.result new file mode 100644 index 00000000000..e9182255bcd --- /dev/null +++ b/storage/spider/mysql-test/spider/r/udf_pushdown.result @@ -0,0 +1,222 @@ +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +child3_1 +child3_2 +child3_3 +# +# MDEV-26545 Spider does not correctly handle UDF and stored function in where conds +# + +##### enable general_log ##### +connection child2_1; +SET @general_log_backup = @@global.general_log; +SET @log_output_backup = @@global.log_output; +SET @@global.general_log = 1; +SET @@global.log_output = "TABLE"; +TRUNCATE TABLE mysql.general_log; + +##### create databases ##### +connection master_1; +CREATE DATABASE auto_test_local; +USE auto_test_local; +connection child2_1; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; + +##### create tables ##### +connection child2_1; +CHILD_CREATE_TABLE +connection master_1; +MASTER_CREATE_TABLE +CREATE TABLE ta_l ( +id INT NOT NULL, +a INT, +PRIMARY KEY(id) +) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1 +INSERT INTO ta_l VALUES +(1, 11), +(2, 22), +(3, 33), +(4, 44), +(5, 55); + +##### create functions ##### +connection master_1; +CREATE FUNCTION `plusone`( param INT ) RETURNS INT +BEGIN +RETURN param + 1; +END // +connection child2_1; +CREATE FUNCTION `plusone`( param INT ) RETURNS INT +BEGIN +RETURN param + 1; +END // + +########## spider_use_pushdown_udf=0 ########## +connection master_1; +SET @@spider_use_pushdown_udf = 0; + +##### test SELECTs ##### +connection master_1; +SELECT * FROM ta_l WHERE id = plusone(1); +id a +2 22 +SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32); +id a +3 33 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%"; +argument +select `id`,`a` from `auto_test_remote`.`ta_r` +select `id`,`a` from `auto_test_remote`.`ta_r` + +##### test UPDATEs ##### +connection master_1; +UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1); +SELECT * FROM ta_l; +id a +1 11 +2 222 +3 33 +4 44 +5 55 +UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32); +SELECT * FROM ta_l; +id a +1 11 +2 222 +3 333 +4 44 +5 55 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%"; +argument +select `id`,`a` from `auto_test_remote`.`ta_r` for update +update `auto_test_remote`.`ta_r` set `a` = 222 where `id` = 2 limit 1 +select `id`,`a` from `auto_test_remote`.`ta_r` for update +update `auto_test_remote`.`ta_r` set `a` = 333 where `id` = 3 and `a` = 33 limit 1 + +##### test DELETEs ##### +connection master_1; +DELETE FROM ta_l WHERE id = plusone(1); +SELECT * FROM ta_l; +id a +1 11 +3 333 +4 44 +5 55 +DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43); +SELECT * FROM ta_l; +id a +1 11 +3 333 +5 55 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%"; +argument +select `id` from `auto_test_remote`.`ta_r` for update +delete from `auto_test_remote`.`ta_r` where `id` = 2 limit 1 +select `id`,`a` from `auto_test_remote`.`ta_r` for update +delete from `auto_test_remote`.`ta_r` where `id` = 4 and `a` = 44 limit 1 + +##### reset records ##### +connection master_1; +TRUNCATE TABLE ta_l; +INSERT INTO ta_l VALUES +(1, 11), +(2, 22), +(3, 33), +(4, 44), +(5, 55); + +########## spider_use_pushdown_udf=1 ########## +connection master_1; +SET @@spider_use_pushdown_udf = 1; + +##### test SELECTs ##### +connection master_1; +SELECT * FROM ta_l WHERE id = plusone(1); +id a +2 22 +SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32); +id a +3 33 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%"; +argument +select `id`,`a` from `auto_test_remote`.`ta_r` where (`id` = (`plusone`(1))) +select `id`,`a` from `auto_test_remote`.`ta_r` where ((`id` in( (`plusone`(1)) , (`plusone`(2)))) and (`a` = (`plusone`(32)))) + +##### test UPDATEs ##### +connection master_1; +UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1); +SELECT * FROM ta_l; +id a +1 11 +2 222 +3 33 +4 44 +5 55 +UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32); +SELECT * FROM ta_l; +id a +1 11 +2 222 +3 333 +4 44 +5 55 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%"; +argument +select `id`,`a` from `auto_test_remote`.`ta_r` where (`id` = (`plusone`(1))) for update +update `auto_test_remote`.`ta_r` set `a` = 222 where `id` = 2 limit 1 +select `id`,`a` from `auto_test_remote`.`ta_r` where ((`id` in( (`plusone`(1)) , (`plusone`(2)))) and (`a` = (`plusone`(32)))) for update +update `auto_test_remote`.`ta_r` set `a` = 333 where `id` = 3 and `a` = 33 limit 1 + +##### test DELETEs ##### +connection master_1; +DELETE FROM ta_l WHERE id = plusone(1); +SELECT * FROM ta_l; +id a +1 11 +3 333 +4 44 +5 55 +DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43); +SELECT * FROM ta_l; +id a +1 11 +3 333 +5 55 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%"; +argument +select `id` from `auto_test_remote`.`ta_r` where (`id` = (`plusone`(1))) for update +delete from `auto_test_remote`.`ta_r` where `id` = 2 limit 1 +select `id`,`a` from `auto_test_remote`.`ta_r` where ((`id` in( (`plusone`(1)) , (`plusone`(2)) , (`plusone`(3)))) and (`a` = (`plusone`(43)))) for update +delete from `auto_test_remote`.`ta_r` where `id` = 4 and `a` = 44 limit 1 + +deinit +connection master_1; +DROP FUNCTION `plusone`; +DROP DATABASE IF EXISTS auto_test_local; +connection child2_1; +SET @@global.general_log = @general_log_backup; +SET @@global.log_output = @log_output_backup; +DROP FUNCTION `plusone`; +DROP DATABASE IF EXISTS auto_test_remote; +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +child3_1 +child3_2 +child3_3 + +end of test diff --git a/storage/spider/mysql-test/spider/t/udf_pushdown.inc b/storage/spider/mysql-test/spider/t/udf_pushdown.inc new file mode 100644 index 00000000000..160e8af21b2 --- /dev/null +++ b/storage/spider/mysql-test/spider/t/udf_pushdown.inc @@ -0,0 +1,48 @@ +--echo +--echo ##### test SELECTs ##### +--connection master_1 +SELECT * FROM ta_l WHERE id = plusone(1); +SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32); + +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%"; + --disable_query_log + TRUNCATE TABLE mysql.general_log; + --enable_query_log +} + +--echo +--echo ##### test UPDATEs ##### +--connection master_1 +UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1); +SELECT * FROM ta_l; +UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32); +SELECT * FROM ta_l; + +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%"; + --disable_query_log + TRUNCATE TABLE mysql.general_log; + --enable_query_log +} + +--echo +--echo ##### test DELETEs ##### +--connection master_1 +DELETE FROM ta_l WHERE id = plusone(1); +SELECT * FROM ta_l; +DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43); +SELECT * FROM ta_l; + +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%"; + --disable_query_log + TRUNCATE TABLE mysql.general_log; + --enable_query_log +} diff --git a/storage/spider/mysql-test/spider/t/udf_pushdown.test b/storage/spider/mysql-test/spider/t/udf_pushdown.test new file mode 100644 index 00000000000..2eadbbbb40b --- /dev/null +++ b/storage/spider/mysql-test/spider/t/udf_pushdown.test @@ -0,0 +1,141 @@ +--disable_warnings +--disable_query_log +--disable_result_log +--source test_init.inc +--enable_result_log +--enable_query_log + +--echo # +--echo # MDEV-26545 Spider does not correctly handle UDF and stored function in where conds +--echo # + +let $CHILD_CREATE_TABLE= + CREATE TABLE ta_r ( + id INT NOT NULL, + a INT, + PRIMARY KEY(id) + ) $CHILD2_1_ENGINE $CHILD2_1_CHARSET; + +let $MASTER_CREATE_TABLE_OUTPUT= + CREATE TABLE ta_l ( + id INT NOT NULL, + a INT, + PRIMARY KEY(id) + ) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1; + +let $MASTER_CREATE_TABLE= + CREATE TABLE ta_l ( + id INT NOT NULL, + a INT, + PRIMARY KEY(id) + ) $MASTER_1_ENGINE $MASTER_1_CHARSET $MASTER_1_COMMENT_2_1; + +--echo +--echo ##### enable general_log ##### +--connection child2_1 +SET @general_log_backup = @@global.general_log; +SET @log_output_backup = @@global.log_output; +SET @@global.general_log = 1; +SET @@global.log_output = "TABLE"; +TRUNCATE TABLE mysql.general_log; + +--echo +--echo ##### create databases ##### +--connection master_1 +CREATE DATABASE auto_test_local; +USE auto_test_local; +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + CREATE DATABASE auto_test_remote; + USE auto_test_remote; +} + +--echo +--echo ##### create tables ##### +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + --disable_query_log + echo CHILD_CREATE_TABLE; + eval $CHILD_CREATE_TABLE; + --enable_query_log +} + +--connection master_1 +--disable_query_log +echo MASTER_CREATE_TABLE; +echo $MASTER_CREATE_TABLE_OUTPUT; +eval $MASTER_CREATE_TABLE; +--enable_query_log + +INSERT INTO ta_l VALUES + (1, 11), + (2, 22), + (3, 33), + (4, 44), + (5, 55); + +--echo +--echo ##### create functions ##### +--connection master_1 +DELIMITER //; +CREATE FUNCTION `plusone`( param INT ) RETURNS INT +BEGIN + RETURN param + 1; +END // +DELIMITER ;// + +--connection child2_1 +DELIMITER //; +CREATE FUNCTION `plusone`( param INT ) RETURNS INT +BEGIN + RETURN param + 1; +END // +DELIMITER ;// + +--echo +--echo ########## spider_use_pushdown_udf=0 ########## +--connection master_1 +SET @@spider_use_pushdown_udf = 0; +--source udf_pushdown.inc + +--echo +--echo ##### reset records ##### +--connection master_1 +TRUNCATE TABLE ta_l; +INSERT INTO ta_l VALUES + (1, 11), + (2, 22), + (3, 33), + (4, 44), + (5, 55); + +--echo +--echo ########## spider_use_pushdown_udf=1 ########## +--connection master_1 +SET @@spider_use_pushdown_udf = 1; +--source udf_pushdown.inc + +--echo +--echo deinit +--disable_warnings +--connection master_1 +DROP FUNCTION `plusone`; +DROP DATABASE IF EXISTS auto_test_local; +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + SET @@global.general_log = @general_log_backup; + SET @@global.log_output = @log_output_backup; + DROP FUNCTION `plusone`; + DROP DATABASE IF EXISTS auto_test_remote; +} +--disable_query_log +--disable_result_log +--source test_deinit.inc +--enable_result_log +--enable_query_log +--enable_warnings +--echo +--echo end of test diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index 1a24d80a95f..4db1232aebf 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -4085,10 +4085,16 @@ int spider_db_mysql_util::open_item_func( separete_str_length = SPIDER_SQL_AND_LEN; } break; + case Item_func::FUNC_SP: case Item_func::UDF_FUNC: use_pushdown_udf = spider_param_use_pushdown_udf(spider->trx->thd, spider->share->use_pushdown_udf); if (!use_pushdown_udf) + /* + This is the default behavior because the remote nodes may deal with + the function in an unexpected way (e.g. not having the same + definition). Users can turn it on if they know what they are doing. + */ DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); if (str) { diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc index c4aaef3a404..468e5844189 100644 --- a/storage/spider/spd_param.cc +++ b/storage/spider/spd_param.cc @@ -1931,7 +1931,7 @@ static MYSQL_THDVAR_INT( "Remote server transmission existence when UDF is used at condition and \"engine_condition_pushdown=1\"", /* comment */ NULL, /* check */ NULL, /* update */ - -1, /* def */ + 0, /* def */ -1, /* min */ 1, /* max */ 0 /* blk */ From 21d19ed45b1b71c2815559a8ad68888a4bfe902f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 21 Sep 2021 18:02:04 +0300 Subject: [PATCH 10/11] Silence a warning about unused Bison label --- sql/sql_yacc.yy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c9dbc6fa8cc..0f6773c899d 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -71,6 +71,9 @@ /* warning C4065: switch statement contains 'default' but no 'case' labels */ #pragma warning (disable : 4065) #endif +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wunused-label" /* yyexhaustedlab: */ +#endif int yylex(void *yylval, void *yythd); From 1cb218c37cc3fe01a1ff2fe9b1cbfb591e90d5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 22 Sep 2021 14:15:00 +0300 Subject: [PATCH 11/11] MDEV-26450: Corruption due to innodb_undo_log_truncate At least since commit 055a3334adc004bd3a897990c2f93178e6bb5f90 (MDEV-13564) the undo log truncation in InnoDB did not work correctly. The main issue is that during the execution of trx_purge_truncate_history() some pages of the newly truncated undo tablespace could be discarded. fsp_try_extend_data_file(): Apply the peculiar rounding of fil_space_t::size_in_header only to the system tablespace, whose size can be expressed in megabytes in a configuration parameter. Other files may freely grow by a number of pages. fseg_alloc_free_page_low(): Do allow the extension of undo tablespaces, and mention the file name in the error message. mtr_t::commit_shrink(): Implement crash-safe shrinking of a tablespace file. First, durably write the log, then shrink the file, and finally release the page latches of the rebuilt tablespace. Refactored from trx_purge_truncate_history(). log_write_and_flush_prepare(), log_write_and_flush(): New functions to durably write log during mtr_t::commit_shrink(). --- .../suite/innodb/r/undo_truncate.result | 22 +--- mysql-test/suite/innodb/t/undo_truncate.test | 64 +---------- storage/innobase/fsp/fsp0fsp.cc | 20 ++-- storage/innobase/include/log0log.h | 14 ++- storage/innobase/include/mtr0mtr.h | 6 +- storage/innobase/include/mtr0mtr.ic | 6 +- storage/innobase/log/log0log.cc | 103 +++++++++++++++++- storage/innobase/mtr/mtr0mtr.cc | 86 ++++++++++++++- storage/innobase/trx/trx0purge.cc | 21 +--- 9 files changed, 219 insertions(+), 123 deletions(-) diff --git a/mysql-test/suite/innodb/r/undo_truncate.result b/mysql-test/suite/innodb/r/undo_truncate.result index 89171d36d0f..67a587b3c1c 100644 --- a/mysql-test/suite/innodb/r/undo_truncate.result +++ b/mysql-test/suite/innodb/r/undo_truncate.result @@ -10,28 +10,12 @@ SET @trunc_start= WHERE variable_name = 'innodb_undo_truncations'); create table t1(keyc int primary key, c char(100)) engine = innodb; create table t2(keyc int primary key, c char(100)) engine = innodb; -CREATE PROCEDURE populate_t1() -BEGIN -DECLARE i INT DEFAULT 1; -while (i <= 20000) DO -insert into t1 values (i, 'a'); -SET i = i + 1; -END WHILE; -END | -CREATE PROCEDURE populate_t2() -BEGIN -DECLARE i INT DEFAULT 1; -while (i <= 20000) DO -insert into t2 values (i, 'a'); -SET i = i + 1; -END WHILE; -END | connect con1,localhost,root,,; begin; -call populate_t1(); +insert into t1 select seq,'a' from seq_1_to_20000; connect con2,localhost,root,,; begin; -call populate_t2(); +insert into t2 select seq,'a' from seq_1_to_20000; connection con1; update t1 set c = 'mysql'; connection con2; @@ -53,8 +37,6 @@ commit; disconnect con2; connection default; drop table t1, t2; -drop PROCEDURE populate_t1; -drop PROCEDURE populate_t2; InnoDB 0 transactions not purged SET GLOBAL innodb_undo_logs = @save_undo_logs; SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency; diff --git a/mysql-test/suite/innodb/t/undo_truncate.test b/mysql-test/suite/innodb/t/undo_truncate.test index 4f350e380ee..9feee84491a 100644 --- a/mysql-test/suite/innodb/t/undo_truncate.test +++ b/mysql-test/suite/innodb/t/undo_truncate.test @@ -5,6 +5,7 @@ # --source include/innodb_page_size.inc --source include/innodb_page_size_small.inc --source include/have_undo_tablespaces.inc +--source include/have_sequence.inc call mtr.add_suppression("InnoDB: The transaction log size is too large"); @@ -27,37 +28,14 @@ WHERE variable_name = 'innodb_undo_truncations'); create table t1(keyc int primary key, c char(100)) engine = innodb; create table t2(keyc int primary key, c char(100)) engine = innodb; # -delimiter |; -CREATE PROCEDURE populate_t1() -BEGIN - DECLARE i INT DEFAULT 1; - while (i <= 20000) DO - insert into t1 values (i, 'a'); - SET i = i + 1; - END WHILE; -END | -delimiter ;| -# -delimiter |; -CREATE PROCEDURE populate_t2() -BEGIN - DECLARE i INT DEFAULT 1; - while (i <= 20000) DO - insert into t2 values (i, 'a'); - SET i = i + 1; - END WHILE; -END | -delimiter ;| -# -# let DATADIR = `select @@datadir`; connect (con1,localhost,root,,); begin; -send call populate_t1(); +send insert into t1 select seq,'a' from seq_1_to_20000; connect (con2,localhost,root,,); begin; -send call populate_t2(); +send insert into t2 select seq,'a' from seq_1_to_20000; connection con1; reap; send update t1 set c = 'mysql'; connection con2; reap; send update t2 set c = 'mysql'; @@ -67,25 +45,12 @@ connection con1; reap; send delete from t1; connection con2; reap; delete from t2; connection con1; reap; -let CHECKFILE = $MYSQL_TMP_DIR/check.txt; -perl; -($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size1) - = stat("$ENV{DATADIR}/undo001"); -($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size2) - = stat("$ENV{DATADIR}/undo002"); -open(OUT, ">$ENV{CHECKFILE}") || die; -print OUT "let \$size1='$size1,$size2';\n"; -close(OUT); -EOF - SET GLOBAL innodb_undo_log_truncate = 1; commit; disconnect con1; connection con2; commit; disconnect con2; connection default; drop table t1, t2; -drop PROCEDURE populate_t1; -drop PROCEDURE populate_t2; --source include/wait_all_purged.inc @@ -100,29 +65,6 @@ if (`select @@innodb_page_size IN (4096,8192,16384)`) source include/wait_condition.inc; } ---source $CHECKFILE -perl; -($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size1) - = stat("$ENV{DATADIR}/undo001"); -($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size2) - = stat("$ENV{DATADIR}/undo002"); -open(OUT, ">$ENV{CHECKFILE}") || die; -print OUT "let \$size2='$size1,$size2';\n"; -close(OUT); -EOF - ---source $CHECKFILE ---remove_file $CHECKFILE - -if ($size1 == $size2) -{ - # This fails for innodb_page_size=64k, occasionally also for 32k. - if (`select @@innodb_page_size IN (4096,8192,16384)`) - { - echo Truncation did not happen: $size1; - } -} - SET GLOBAL innodb_undo_logs = @save_undo_logs; SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency; SET GLOBAL innodb_undo_log_truncate = @save_truncate; diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 43989d57db8..064eb133ebc 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -956,11 +956,13 @@ fsp_try_extend_data_file(fil_space_t* space, fsp_header_t* header, mtr_t* mtr) return(0); } - /* We ignore any fragments of a full megabyte when storing the size - to the space header */ + /* For the system tablespace, we ignore any fragments of a + full megabyte when storing the size to the space header */ - space->size_in_header = ut_2pow_round( - space->size, (1024 * 1024) / page_size.physical()); + space->size_in_header = space->id + ? space->size + : ut_2pow_round(space->size, + (1024 * 1024) / page_size.physical()); mlog_write_ulint( header + FSP_SIZE, space->size_in_header, MLOG_4BYTES, mtr); @@ -1392,7 +1394,7 @@ fsp_alloc_free_page( /* It must be that we are extending a single-table tablespace whose size is still < 64 pages */ - ut_a(!is_system_tablespace(space_id)); + ut_a(!is_predefined_tablespace(space_id)); if (page_no >= FSP_EXTENT_SIZE) { ib::error() << "Trying to extend a single-table" " tablespace " << space->name << " , by single" @@ -2514,14 +2516,14 @@ take_hinted_page: return(NULL); } - if (space->size <= ret_page && !is_system_tablespace(space_id)) { + if (space->size <= ret_page && !is_predefined_tablespace(space_id)) { /* It must be that we are extending a single-table tablespace whose size is still < 64 pages */ if (ret_page >= FSP_EXTENT_SIZE) { - ib::error() << "Error (2): trying to extend" - " a single-table tablespace " << space_id - << " by single page(s) though the" + ib::error() << "Trying to extend '" + << space->chain.start->name + << "' by single page(s) though the" << " space size " << space->size << ". Page no " << ret_page << "."; ut_ad(!has_done_reservation); diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 612a27976e7..c0b92fb7497 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -189,9 +189,15 @@ log_write_up_to( /** write to the log file up to the last log entry. @param[in] sync whether we want the written log also to be flushed to disk. */ -void -log_buffer_flush_to_disk( - bool sync = true); +void log_buffer_flush_to_disk(bool sync= true); + + +/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.mutex. */ +#define log_write_and_flush_prepare() log_write_mutex_enter() + +/** Durably write the log up to log_sys.lsn and release log_sys.mutex. */ +ATTRIBUTE_COLD void log_write_and_flush(); + /****************************************************************//** This functions writes the log buffer to the log file and if 'flush' is set it forces a flush of the log file as well. This is meant to be diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index b57a38f8eab..6d729beab12 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2020, MariaDB Corporation. +Copyright (c) 2013, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -161,6 +161,10 @@ struct mtr_t { /** Commit the mini-transaction. */ void commit(); + /** Commit a mini-transaction that is shrinking a tablespace. + @param space tablespace that is being shrunk */ + ATTRIBUTE_COLD void commit_shrink(fil_space_t &space); + /** Commit a mini-transaction that did not modify any pages, but generated some redo log on a higher level, such as MLOG_FILE_NAME records and a MLOG_CHECKPOINT marker. diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic index a45d088d5d7..24477689db8 100644 --- a/storage/innobase/include/mtr0mtr.ic +++ b/storage/innobase/include/mtr0mtr.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,8 +53,8 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type) /* If this mtr has x-fixed a clean page then we set the made_dirty flag. This tells us if we need to - grab log_flush_order_mutex at mtr_commit so that we - can insert the dirtied page to the flush list. */ + grab log_sys.flush_order_mutex at mtr_t::commit() so that we + can insert the dirtied page into the flush list. */ if ((type == MTR_MEMO_PAGE_X_FIX || type == MTR_MEMO_PAGE_SX_FIX) && !m_made_dirty) { diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 4c68f3743e9..945c97daf4a 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2014, 2020, MariaDB Corporation. +Copyright (c) 2014, 2021, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1247,12 +1247,103 @@ loop: /** write to the log file up to the last log entry. @param[in] sync whether we want the written log also to be flushed to disk. */ -void -log_buffer_flush_to_disk( - bool sync) +void log_buffer_flush_to_disk(bool sync) { - ut_ad(!srv_read_only_mode); - log_write_up_to(log_get_lsn(), sync); + ut_ad(!srv_read_only_mode); + log_write_up_to(log_get_lsn(), sync); +} + + +/** Durably write the log and release log_sys.mutex */ +ATTRIBUTE_COLD void log_write_and_flush() +{ + ut_ad(!srv_read_only_mode); + ut_ad(!recv_no_log_write); + ut_ad(!recv_recovery_is_on()); + + /* The following code is adapted from log_write_up_to(). */ + DBUG_PRINT("ib_log", ("write " LSN_PF " to " LSN_PF, + log_sys->write_lsn, log_sys->lsn)); + log_sys->n_pending_flushes++; + log_sys->current_flush_lsn= log_sys->lsn; + os_event_reset(log_sys->flush_event); + ut_ad(log_sys->buf_free != log_sys->buf_next_to_write); + ulint start_offset= log_sys->buf_next_to_write; + ulint end_offset= log_sys->buf_free; + ulint area_start= ut_2pow_round(start_offset, ulint(OS_FILE_LOG_BLOCK_SIZE)); + ulint area_end= ut_calc_align(end_offset, ulint(OS_FILE_LOG_BLOCK_SIZE)); + ulong write_ahead_size= srv_log_write_ahead_size; + + log_block_set_flush_bit(log_sys->buf + area_start, TRUE); + log_block_set_checkpoint_no(log_sys->buf + area_end - OS_FILE_LOG_BLOCK_SIZE, + log_sys->next_checkpoint_no); + lsn_t write_lsn= log_sys->lsn; + byte *write_buf= log_sys->buf; + + ut_ad(area_end - area_start > 0); + + log_buffer_switch(); + + log_group_set_fields(&log_sys->log, log_sys->write_lsn); + + /* Erase the end of the last log block. */ + memset(write_buf + end_offset, 0, + ~end_offset & (OS_FILE_LOG_BLOCK_SIZE - 1)); + /* Calculate pad_size if needed. */ + ulint pad_size= 0; + if (write_ahead_size > OS_FILE_LOG_BLOCK_SIZE) + { + lsn_t end_offset= + log_group_calc_lsn_offset(ut_uint64_align_up(write_lsn, + OS_FILE_LOG_BLOCK_SIZE), + &log_sys->log); + ulint end_offset_in_unit= (ulint) (end_offset % write_ahead_size); + + if (end_offset_in_unit && (area_end - area_start) > end_offset_in_unit) + { + /* The first block in the unit was initialized after the last + writing. Needs to be written padded data once. */ + pad_size= std::min(ulint(write_ahead_size) - end_offset_in_unit, + log_sys->buf_size - area_end); + memset(write_buf + area_end, 0, pad_size); + } + } + + if (log_sys->is_encrypted()) + log_crypt(write_buf + area_start, log_sys->write_lsn, + area_end - area_start); + + /* Do the write to the log files */ + log_group_write_buf(&log_sys->log, write_buf + area_start, + area_end - area_start + pad_size, +#ifdef UNIV_DEBUG + pad_size, +#endif /* UNIV_DEBUG */ + ut_uint64_align_down(log_sys->write_lsn, + OS_FILE_LOG_BLOCK_SIZE), + start_offset - area_start); + srv_stats.log_padded.add(pad_size); + log_sys->write_lsn= write_lsn; + + log_write_mutex_exit(); + + /* Code adapted from log_write_flush_to_disk_low() */ + + ut_a(log_sys->n_pending_flushes == 1); /* No other threads here */ + + if (srv_file_flush_method != SRV_O_DSYNC) + fil_flush(SRV_LOG_SPACE_FIRST_ID); + + log_sys->flushed_to_disk_lsn= log_sys->current_flush_lsn; + + log_sys->n_pending_flushes--; + + os_event_set(log_sys->flush_event); + + lsn_t wrote_lsn= log_sys->write_lsn, flush_lsn= log_sys->flushed_to_disk_lsn; + log_mutex_exit(); + + innobase_mysql_log_notify(wrote_lsn, flush_lsn); } /****************************************************************//** diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index fefc0687ddb..4bd90e7842d 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -465,6 +465,90 @@ mtr_t::commit() release_resources(); } +#ifdef UNIV_DEBUG +/** Check that all pages belong to a shrunk tablespace. */ +struct Shrink +{ + const fil_space_t &space; + Shrink(const fil_space_t &space) : space(space) {} + + bool operator()(const mtr_memo_slot_t *slot) const + { + if (!slot->object) + return true; + switch (slot->type) { + default: + ut_ad("invalid type" == 0); + return false; + case MTR_MEMO_MODIFY: + break; + case MTR_MEMO_SPACE_X_LOCK: + ut_ad(&space == slot->object); + return true; + case MTR_MEMO_PAGE_X_FIX: + case MTR_MEMO_PAGE_SX_FIX: + const buf_page_t &bpage= static_cast(slot->object)->page; + const page_id_t &id= bpage.id; + if (id.space() == 0 && id.page_no() == TRX_SYS_PAGE_NO) + { + ut_ad(srv_is_undo_tablespace(space.id)); + break; + } + ut_ad(id.space() == space.id); + ut_ad(id.page_no() < space.size); + ut_ad(bpage.state == BUF_BLOCK_FILE_PAGE); + ut_ad(!bpage.oldest_modification); + break; + } + return true; + } +}; +#endif + +/** Commit a mini-transaction that is shrinking a tablespace. +@param space tablespace that is being shrunk */ +void mtr_t::commit_shrink(fil_space_t &space) +{ + ut_ad(is_active()); + ut_ad(!is_inside_ibuf()); + ut_ad(!high_level_read_only); + ut_ad(m_modifications); + ut_ad(m_made_dirty); + ut_ad(!recv_recovery_is_on()); + ut_ad(m_log_mode == MTR_LOG_ALL); + ut_ad(UT_LIST_GET_LEN(space.chain) == 1); + + log_write_and_flush_prepare(); + + const lsn_t start_lsn= finish_write(prepare_write()); + + log_flush_order_mutex_enter(); + /* Durably write the reduced FSP_SIZE before truncating the data file. */ + log_write_and_flush(); + + os_file_truncate(space.chain.start->name, space.chain.start->handle, + os_offset_t(space.size) << srv_page_size_shift, true); + + ut_d(m_memo.for_each_block_in_reverse(CIterate(space))); + + m_memo.for_each_block_in_reverse(CIterate + (ReleaseBlocks(start_lsn, m_commit_lsn, + m_flush_observer))); + log_flush_order_mutex_exit(); + + mutex_enter(&fil_system->mutex); + ut_ad(space.is_being_truncated); + ut_ad(space.stop_new_ops); + space.stop_new_ops= false; + space.is_being_truncated= false; + mutex_exit(&fil_system->mutex); + + m_memo.for_each_block_in_reverse(CIterate()); + srv_stats.log_write_requests.inc(); + + release_resources(); +} + /** Commit a mini-transaction that did not modify any pages, but generated some redo log on a higher level, such as MLOG_FILE_NAME records and a MLOG_CHECKPOINT marker. diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 2025ac70beb..a9593807d53 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1062,26 +1062,11 @@ trx_purge_initiate_truncate( rseg->last_del_marks = FALSE; } - mtr.commit(); - /* Write-ahead the redo log record. */ - log_write_up_to(mtr.commit_lsn(), true); + mtr.commit_shrink(*space); - /* Trim the file size. */ - os_file_truncate(file->name, file->handle, - os_offset_t(size) << srv_page_size_shift, true); - - /* This is only executed by the srv_purge_coordinator_thread. */ + /* No mutex; this is only updated by the purge coordinator. */ export_vars.innodb_undo_truncations++; - /* TODO: PUNCH_HOLE the garbage (with write-ahead logging) */ - - mutex_enter(&fil_system->mutex); - ut_ad(space->stop_new_ops); - ut_ad(space->is_being_truncated); - space->stop_new_ops = false; - space->is_being_truncated = false; - mutex_exit(&fil_system->mutex); - if (purge_sys->rseg != NULL && purge_sys->rseg->last_page_no == FIL_NULL) { /* If purge_sys->rseg is pointing to rseg that was recently