From 516f7c111b0cc3e9ad193c5a4e8c9171a081a362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 25 Sep 2019 11:29:21 +0300 Subject: [PATCH 1/5] Speed up main.sum_distinct-big Eliminate one InnoDB table with 128*16384 rows, and use the sequence engine instead. Also, run everything in a single transaction, to prevent purge from running concurrently unnecessarily. (Starting with MariaDB Server 10.3, purge would reset the DB_TRX_ID after INSERT.) --- mysql-test/r/sum_distinct-big.result | 40 +++++------------------ mysql-test/t/sum_distinct-big.test | 48 ++++++---------------------- 2 files changed, 17 insertions(+), 71 deletions(-) diff --git a/mysql-test/r/sum_distinct-big.result b/mysql-test/r/sum_distinct-big.result index 2d350826ac8..13e3e62fed2 100644 --- a/mysql-test/r/sum_distinct-big.result +++ b/mysql-test/r/sum_distinct-big.result @@ -1,7 +1,5 @@ -DROP TABLE IF EXISTS t1, t2; set @save_tmp_table_size=@@tmp_table_size; set @save_max_heap_table_size=@@max_heap_table_size; -set @save_storage_engine=@@storage_engine; set storage_engine=MYISAM; CREATE TABLE t1 (id INTEGER); CREATE TABLE t2 (id INTEGER); @@ -126,35 +124,13 @@ DROP TABLE t1; DROP TABLE t2; SET @@tmp_table_size=@save_tmp_table_size; SET @@max_heap_table_size=@save_max_heap_table_size; -# -# Bug mdev-4311: COUNT(DISTINCT...) requiring a file for Unique -# (bug #68749) -# -set @save_storage_engine=@@storage_engine; -set storage_engine=INNODB; -CREATE TABLE t1 (id INTEGER) ENGINE=InnoDB; +# +# MDEV-4311: COUNT(DISTINCT...) requiring a file for UNIQUE (bug #68749) +# CREATE TABLE t2 (id INTEGER) ENGINE=InnoDB; -INSERT INTO t1 (id) VALUES (1), (1), (1),(1); -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 SELECT id+1 FROM t1; -INSERT INTO t1 SELECT id+2 FROM t1; -INSERT INTO t1 SELECT id+4 FROM t1; -INSERT INTO t1 SELECT id+8 FROM t1; -INSERT INTO t1 SELECT id+16 FROM t1; -INSERT INTO t1 SELECT id+32 FROM t1; -INSERT INTO t1 SELECT id+64 FROM t1; -INSERT INTO t1 SELECT id+128 FROM t1; -INSERT INTO t1 SELECT id+256 FROM t1; -INSERT INTO t1 SELECT id+512 FROM t1; -INSERT INTO t1 SELECT id+1024 FROM t1; -INSERT INTO t1 SELECT id+2048 FROM t1; -INSERT INTO t1 SELECT id+4096 FROM t1; -INSERT INTO t1 SELECT id+8192 FROM t1; -INSERT INTO t2 SELECT id FROM t1 ORDER BY id*rand(); +BEGIN; +INSERT INTO t2 SELECT b.seq FROM seq_1_to_128 a, seq_1_to_16384 b +ORDER BY b.seq*rand(); INSERT INTO t2 VALUE(NULL); # With default tmp_table_size / max_heap_table_size SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; @@ -176,5 +152,5 @@ SET @@max_heap_table_size=@save_max_heap_table_size; SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; sm 16384 -DROP TABLE t1,t2; -set storage_engine=@save_storage_engine; +COMMIT; +DROP TABLE t2; diff --git a/mysql-test/t/sum_distinct-big.test b/mysql-test/t/sum_distinct-big.test index fee406ee46d..bc318bf624d 100644 --- a/mysql-test/t/sum_distinct-big.test +++ b/mysql-test/t/sum_distinct-big.test @@ -4,15 +4,11 @@ --source include/big_test.inc --source include/have_innodb.inc ---disable_warnings -DROP TABLE IF EXISTS t1, t2; ---enable_warnings +--source include/have_sequence.inc set @save_tmp_table_size=@@tmp_table_size; set @save_max_heap_table_size=@@max_heap_table_size; -set @save_storage_engine=@@storage_engine; - # # Test the case when distinct values doesn't fit in memory and # filesort is used (see uniques.cc:merge_walk) @@ -94,39 +90,14 @@ DROP TABLE t2; SET @@tmp_table_size=@save_tmp_table_size; SET @@max_heap_table_size=@save_max_heap_table_size; ---echo # ---echo # Bug mdev-4311: COUNT(DISTINCT...) requiring a file for Unique ---echo # (bug #68749) ---echo # - -set @save_storage_engine=@@storage_engine; -set storage_engine=INNODB; - -CREATE TABLE t1 (id INTEGER) ENGINE=InnoDB; +--echo # +--echo # MDEV-4311: COUNT(DISTINCT...) requiring a file for UNIQUE (bug #68749) +--echo # CREATE TABLE t2 (id INTEGER) ENGINE=InnoDB; -INSERT INTO t1 (id) VALUES (1), (1), (1),(1); -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 SELECT id+1 FROM t1; -INSERT INTO t1 SELECT id+2 FROM t1; -INSERT INTO t1 SELECT id+4 FROM t1; -INSERT INTO t1 SELECT id+8 FROM t1; -INSERT INTO t1 SELECT id+16 FROM t1; -INSERT INTO t1 SELECT id+32 FROM t1; -INSERT INTO t1 SELECT id+64 FROM t1; -INSERT INTO t1 SELECT id+128 FROM t1; -INSERT INTO t1 SELECT id+256 FROM t1; -INSERT INTO t1 SELECT id+512 FROM t1; -INSERT INTO t1 SELECT id+1024 FROM t1; -INSERT INTO t1 SELECT id+2048 FROM t1; -INSERT INTO t1 SELECT id+4096 FROM t1; -INSERT INTO t1 SELECT id+8192 FROM t1; - -INSERT INTO t2 SELECT id FROM t1 ORDER BY id*rand(); +BEGIN; +INSERT INTO t2 SELECT b.seq FROM seq_1_to_128 a, seq_1_to_16384 b +ORDER BY b.seq*rand(); INSERT INTO t2 VALUE(NULL); --echo # With default tmp_table_size / max_heap_table_size @@ -147,7 +118,6 @@ SET @@max_heap_table_size=@save_max_heap_table_size; --echo # Back to default tmp_table_size / max_heap_table_size SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; +COMMIT; -DROP TABLE t1,t2; - -set storage_engine=@save_storage_engine; +DROP TABLE t2; From 23d675453f182becb3bc5811bfa7e2e310cbed1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 12:44:04 +0300 Subject: [PATCH 2/5] MDEV-20675 Crash in SHOW ENGINE INNODB STATUS with innodb_force_recovery=5 Add a test case. MariaDB Server 10.2 is not affected. --- mysql-test/suite/innodb/t/innodb_force_recovery.test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mysql-test/suite/innodb/t/innodb_force_recovery.test b/mysql-test/suite/innodb/t/innodb_force_recovery.test index b5b48281161..9d7cf9d80b7 100644 --- a/mysql-test/suite/innodb/t/innodb_force_recovery.test +++ b/mysql-test/suite/innodb/t/innodb_force_recovery.test @@ -19,6 +19,7 @@ SET GLOBAL innodb_fast_shutdown = 0; --echo # Restart the server with innodb_force_recovery as 4. --let $restart_parameters= --innodb-force-recovery=4 --source include/restart_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; select * from t1; @@ -58,6 +59,7 @@ show tables; --echo # Restart the server with innodb_force_recovery as 5. --let $restart_parameters= --innodb-force-recovery=5 --source include/restart_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; select * from t2; @@ -98,6 +100,7 @@ show tables; --echo # Restart the server with innodb_force_recovery as 6. --let $restart_parameters= --innodb-force-recovery=6 --source include/restart_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; select * from t2; @@ -135,6 +138,7 @@ show tables; --echo # Restart the server with innodb_force_recovery=2 --let $restart_parameters= --innodb-force-recovery=2 --source include/restart_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; select * from t2; begin; @@ -152,6 +156,7 @@ connection default; --echo # Restart the server with innodb_force_recovery=3 --let $restart_parameters= --innodb-force-recovery=3 --source include/start_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; select * from t2; From 2d6719d7ee92843d5b0b9a27c7deaff5cacd4745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 12:48:55 +0300 Subject: [PATCH 3/5] MDEV-19514 preparation: Extend innodb.innodb-change-buffer-recovery Test innodb_read_only startup (which will be refused after a crash), and test also innodb_force_recovery=5, and extract some change buffer merge statistics. Omit any statistics about delete (purge) buffering, because purge could happen at any time. Use the sequence storage engine for populating the table. --- .../r/innodb-change-buffer-recovery.result | 36 +++++++++------- .../innodb-change-buffer-recovery-master.opt | 1 - .../t/innodb-change-buffer-recovery.test | 43 +++++++++++-------- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result b/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result index f03072053c3..156ef0e56be 100644 --- a/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result +++ b/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result @@ -2,6 +2,10 @@ # Bug#69122 - INNODB DOESN'T REDO-LOG INSERT BUFFER MERGE # OPERATION IF IT IS DONE IN-PLACE # +call mtr.add_suppression("InnoDB: innodb_read_only prevents crash recovery"); +call mtr.add_suppression("Plugin initialization aborted at srv0start\\.cc"); +call mtr.add_suppression("Plugin 'InnoDB'"); +FLUSH TABLES; CREATE TABLE t1( a INT AUTO_INCREMENT PRIMARY KEY, b CHAR(1), @@ -9,20 +13,7 @@ c INT, INDEX(b)) ENGINE=InnoDB STATS_PERSISTENT=0; SET GLOBAL innodb_change_buffering_debug = 1; -INSERT INTO t1 VALUES(0,'x',1); -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; +INSERT INTO t1 SELECT 0,'x',1 FROM seq_1_to_8192; BEGIN; SELECT b FROM t1 LIMIT 3; b @@ -38,8 +29,23 @@ SELECT b FROM t1 LIMIT 3; ERROR HY000: Lost connection to MySQL server during query disconnect con1; connection default; -FOUND 1 /Wrote log record for ibuf update in place operation/ in my_restart.err +FOUND 1 /Wrote log record for ibuf update in place operation/ in mysqld.1.err +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check Error Unknown storage engine 'InnoDB' +test.t1 check error Corrupt +FOUND 1 /innodb_read_only prevents crash recovery/ in mysqld.1.err +SELECT * FROM t1 LIMIT 1; +a b c +1 X 1 +SHOW ENGINE INNODB STATUS; +Type Name Status +InnoDB insert 0, delete mark 0 +SET GLOBAL innodb_fast_shutdown=0; CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK +SHOW ENGINE INNODB STATUS; +Type Name Status +InnoDB insert 79, delete mark 1 DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery-master.opt b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery-master.opt index 97b259ee047..e5d7090c883 100644 --- a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery-master.opt +++ b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery-master.opt @@ -1,2 +1 @@ ---log-error=$MYSQLTEST_VARDIR/tmp/my_restart.err --innodb_buffer_pool_size=24M diff --git a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test index d8cc21c550c..79d9cc814a0 100644 --- a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test +++ b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test @@ -11,6 +11,12 @@ --source include/not_valgrind.inc # This test is slow on buildbot. --source include/big_test.inc +--source include/have_sequence.inc + +call mtr.add_suppression("InnoDB: innodb_read_only prevents crash recovery"); +call mtr.add_suppression("Plugin initialization aborted at srv0start\\.cc"); +call mtr.add_suppression("Plugin 'InnoDB'"); +FLUSH TABLES; CREATE TABLE t1( a INT AUTO_INCREMENT PRIMARY KEY, @@ -27,25 +33,12 @@ ENGINE=InnoDB STATS_PERSISTENT=0; # change buffering is possible, so that the change buffer will be used # whenever possible. SET GLOBAL innodb_change_buffering_debug = 1; -let SEARCH_FILE = $MYSQLTEST_VARDIR/tmp/my_restart.err; +let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err; # Create enough rows for the table, so that the change buffer will be # used for modifying the secondary index page. There must be multiple # index pages, because changes to the root page are never buffered. -INSERT INTO t1 VALUES(0,'x',1); -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; +INSERT INTO t1 SELECT 0,'x',1 FROM seq_1_to_8192; BEGIN; SELECT b FROM t1 LIMIT 3; @@ -63,10 +56,26 @@ SET DEBUG_DBUG='+d,crash_after_log_ibuf_upd_inplace'; SELECT b FROM t1 LIMIT 3; disconnect con1; connection default; - let SEARCH_PATTERN=Wrote log record for ibuf update in place operation; --source include/search_pattern_in_file.inc ---source include/start_mysqld.inc +--let $restart_parameters= --innodb-read-only +--source include/start_mysqld.inc CHECK TABLE t1; +--source include/shutdown_mysqld.inc +let SEARCH_PATTERN=innodb_read_only prevents crash recovery; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --innodb-force-recovery=5 +--source include/start_mysqld.inc +SELECT * FROM t1 LIMIT 1; +replace_regex /.*operations:.* (insert.*), delete \d.*discarded .*/\1/; +SHOW ENGINE INNODB STATUS; +# Slow shutdown will not merge the changes due to innodb_force_recovery=5. +SET GLOBAL innodb_fast_shutdown=0; +--let $restart_parameters= +--source include/restart_mysqld.inc +CHECK TABLE t1; +replace_regex /.*operations:.* (insert.*), delete \d.*discarded .*/\1/; +SHOW ENGINE INNODB STATUS; DROP TABLE t1; From 3e4931cdf3485a8762235401bd8ca3128ab31a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 13:18:22 +0300 Subject: [PATCH 4/5] MDEV-20675 Crash in SHOW ENGINE INNODB STATUS with innodb_force_recovery=5 lock_print_info::operator(): Do not dereference purge_sys.query in case it is NULL. We would not initialize purge_sys if innodb_force_recovery is set to 5 or 6. The test case will be added by merge from 10.2. --- storage/innobase/lock/lock0lock.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index f4a57e4955a..28055676f74 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4684,12 +4684,15 @@ lock_trx_print_locks( /** Functor to display all transactions */ struct lock_print_info { - lock_print_info(FILE* file, time_t now) : file(file), now(now) {} + lock_print_info(FILE* file, time_t now) : + file(file), now(now), + purge_trx(purge_sys.query ? purge_sys.query->trx : NULL) + {} void operator()(const trx_t* trx) const { ut_ad(mutex_own(&trx_sys.mutex)); - if (trx == purge_sys.query->trx) + if (UNIV_UNLIKELY(trx == purge_trx)) return; lock_trx_print_wait_and_mvcc_state(file, trx, now); @@ -4699,6 +4702,7 @@ struct lock_print_info FILE* const file; const time_t now; + const trx_t* const purge_trx; }; /*********************************************************************//** From 46facaedbffe8031f249fb7ecfe114273023fc66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 15:18:43 +0300 Subject: [PATCH 5/5] Fix GCC 9 -Wmaybe-uninitialized Always initialize ScopedStatementReplication::saved_binlog_format, so that GCC cannot emit a bogus warning about ScopedStatementReplication::~ScopedStatementReplication() using the variable. The code was originally introduced in commit d998da03069dca1aae83ec6af7eb407bbdc9fc58. --- sql/sql_class.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sql/sql_class.h b/sql/sql_class.h index 9bd486a540e..19f7265b24d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2016, Oracle and/or its affiliates. - Copyright (c) 2009, 2017, MariaDB Corporation. + Copyright (c) 2009, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -6761,11 +6761,12 @@ void dbug_serve_apcs(THD *thd, int n_calls); class ScopedStatementReplication { public: - ScopedStatementReplication(THD *thd) : thd(thd) - { - if (thd) - saved_binlog_format= thd->set_current_stmt_binlog_format_stmt(); - } + ScopedStatementReplication(THD *thd) : + saved_binlog_format(thd + ? thd->set_current_stmt_binlog_format_stmt() + : BINLOG_FORMAT_MIXED), + thd(thd) + {} ~ScopedStatementReplication() { if (thd) @@ -6773,8 +6774,8 @@ public: } private: - enum_binlog_format saved_binlog_format; - THD *thd; + const enum_binlog_format saved_binlog_format; + THD *const thd; }; #endif /* MYSQL_SERVER */