From 516f7c111b0cc3e9ad193c5a4e8c9171a081a362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 25 Sep 2019 11:29:21 +0300 Subject: [PATCH 1/7] Speed up main.sum_distinct-big Eliminate one InnoDB table with 128*16384 rows, and use the sequence engine instead. Also, run everything in a single transaction, to prevent purge from running concurrently unnecessarily. (Starting with MariaDB Server 10.3, purge would reset the DB_TRX_ID after INSERT.) --- mysql-test/r/sum_distinct-big.result | 40 +++++------------------ mysql-test/t/sum_distinct-big.test | 48 ++++++---------------------- 2 files changed, 17 insertions(+), 71 deletions(-) diff --git a/mysql-test/r/sum_distinct-big.result b/mysql-test/r/sum_distinct-big.result index 2d350826ac8..13e3e62fed2 100644 --- a/mysql-test/r/sum_distinct-big.result +++ b/mysql-test/r/sum_distinct-big.result @@ -1,7 +1,5 @@ -DROP TABLE IF EXISTS t1, t2; set @save_tmp_table_size=@@tmp_table_size; set @save_max_heap_table_size=@@max_heap_table_size; -set @save_storage_engine=@@storage_engine; set storage_engine=MYISAM; CREATE TABLE t1 (id INTEGER); CREATE TABLE t2 (id INTEGER); @@ -126,35 +124,13 @@ DROP TABLE t1; DROP TABLE t2; SET @@tmp_table_size=@save_tmp_table_size; SET @@max_heap_table_size=@save_max_heap_table_size; -# -# Bug mdev-4311: COUNT(DISTINCT...) requiring a file for Unique -# (bug #68749) -# -set @save_storage_engine=@@storage_engine; -set storage_engine=INNODB; -CREATE TABLE t1 (id INTEGER) ENGINE=InnoDB; +# +# MDEV-4311: COUNT(DISTINCT...) requiring a file for UNIQUE (bug #68749) +# CREATE TABLE t2 (id INTEGER) ENGINE=InnoDB; -INSERT INTO t1 (id) VALUES (1), (1), (1),(1); -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 SELECT id+1 FROM t1; -INSERT INTO t1 SELECT id+2 FROM t1; -INSERT INTO t1 SELECT id+4 FROM t1; -INSERT INTO t1 SELECT id+8 FROM t1; -INSERT INTO t1 SELECT id+16 FROM t1; -INSERT INTO t1 SELECT id+32 FROM t1; -INSERT INTO t1 SELECT id+64 FROM t1; -INSERT INTO t1 SELECT id+128 FROM t1; -INSERT INTO t1 SELECT id+256 FROM t1; -INSERT INTO t1 SELECT id+512 FROM t1; -INSERT INTO t1 SELECT id+1024 FROM t1; -INSERT INTO t1 SELECT id+2048 FROM t1; -INSERT INTO t1 SELECT id+4096 FROM t1; -INSERT INTO t1 SELECT id+8192 FROM t1; -INSERT INTO t2 SELECT id FROM t1 ORDER BY id*rand(); +BEGIN; +INSERT INTO t2 SELECT b.seq FROM seq_1_to_128 a, seq_1_to_16384 b +ORDER BY b.seq*rand(); INSERT INTO t2 VALUE(NULL); # With default tmp_table_size / max_heap_table_size SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; @@ -176,5 +152,5 @@ SET @@max_heap_table_size=@save_max_heap_table_size; SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; sm 16384 -DROP TABLE t1,t2; -set storage_engine=@save_storage_engine; +COMMIT; +DROP TABLE t2; diff --git a/mysql-test/t/sum_distinct-big.test b/mysql-test/t/sum_distinct-big.test index fee406ee46d..bc318bf624d 100644 --- a/mysql-test/t/sum_distinct-big.test +++ b/mysql-test/t/sum_distinct-big.test @@ -4,15 +4,11 @@ --source include/big_test.inc --source include/have_innodb.inc ---disable_warnings -DROP TABLE IF EXISTS t1, t2; ---enable_warnings +--source include/have_sequence.inc set @save_tmp_table_size=@@tmp_table_size; set @save_max_heap_table_size=@@max_heap_table_size; -set @save_storage_engine=@@storage_engine; - # # Test the case when distinct values doesn't fit in memory and # filesort is used (see uniques.cc:merge_walk) @@ -94,39 +90,14 @@ DROP TABLE t2; SET @@tmp_table_size=@save_tmp_table_size; SET @@max_heap_table_size=@save_max_heap_table_size; ---echo # ---echo # Bug mdev-4311: COUNT(DISTINCT...) requiring a file for Unique ---echo # (bug #68749) ---echo # - -set @save_storage_engine=@@storage_engine; -set storage_engine=INNODB; - -CREATE TABLE t1 (id INTEGER) ENGINE=InnoDB; +--echo # +--echo # MDEV-4311: COUNT(DISTINCT...) requiring a file for UNIQUE (bug #68749) +--echo # CREATE TABLE t2 (id INTEGER) ENGINE=InnoDB; -INSERT INTO t1 (id) VALUES (1), (1), (1),(1); -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 (id) SELECT id FROM t1; -INSERT INTO t1 SELECT id+1 FROM t1; -INSERT INTO t1 SELECT id+2 FROM t1; -INSERT INTO t1 SELECT id+4 FROM t1; -INSERT INTO t1 SELECT id+8 FROM t1; -INSERT INTO t1 SELECT id+16 FROM t1; -INSERT INTO t1 SELECT id+32 FROM t1; -INSERT INTO t1 SELECT id+64 FROM t1; -INSERT INTO t1 SELECT id+128 FROM t1; -INSERT INTO t1 SELECT id+256 FROM t1; -INSERT INTO t1 SELECT id+512 FROM t1; -INSERT INTO t1 SELECT id+1024 FROM t1; -INSERT INTO t1 SELECT id+2048 FROM t1; -INSERT INTO t1 SELECT id+4096 FROM t1; -INSERT INTO t1 SELECT id+8192 FROM t1; - -INSERT INTO t2 SELECT id FROM t1 ORDER BY id*rand(); +BEGIN; +INSERT INTO t2 SELECT b.seq FROM seq_1_to_128 a, seq_1_to_16384 b +ORDER BY b.seq*rand(); INSERT INTO t2 VALUE(NULL); --echo # With default tmp_table_size / max_heap_table_size @@ -147,7 +118,6 @@ SET @@max_heap_table_size=@save_max_heap_table_size; --echo # Back to default tmp_table_size / max_heap_table_size SELECT SQL_NO_CACHE count(DISTINCT id) sm FROM t2; +COMMIT; -DROP TABLE t1,t2; - -set storage_engine=@save_storage_engine; +DROP TABLE t2; From e3c39c0be82e69acb813e7c5e0f2aea7d5546e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 10:25:34 +0300 Subject: [PATCH 2/7] MDEV-13564 follow-up: Remove dead code In MariaDB 10.4.0, commit 09af00cbde1d62dfda574dee10e5c0fd240c3f7f removed the crash-upgrade logic for the MariaDB 10.2 innodb_safe_truncate=OFF TRUNCATE TABLE (which was the only option between MariaDB 10.2.2 and 10.2.18), but failed to adjust some comments and code. buf_page_io_complete(): Remove a bogus comment about TRUNCATE. dict_recreate_index_tree(): Unused function; remove. fil_space_t::stop_new_ops: Clarify the comment. fil_space_acquire_low(): Remove a bogus comment about TRUNCATE. fil_check_pending_ops(), fil_check_pending_io(): Adjust a warning message. This code is only invoked as part of DISCARD TABLESPACE or DROP TABLE. DROP TABLE is internally used as part of ALTER TABLE, OPTIMIZE TABLE, or TRUNCATE TABLE. RemoteDatafile::create_link_file(): Clarify a comment. ibuf_delete_for_discarded_space(): Clarify the function comment. dict_table_x_lock_indexes(), dict_table_x_unlock_indexes(): Merge with the only remaining caller, row_quiesce_set_state(). page_create_zip(): Remove a bogus comment about TRUNCATE. --- storage/innobase/buf/buf0buf.cc | 2 - storage/innobase/dict/dict0crea.cc | 81 --------------------------- storage/innobase/fil/fil0fil.cc | 14 ++--- storage/innobase/fsp/fsp0file.cc | 5 +- storage/innobase/ibuf/ibuf0ibuf.cc | 13 ++--- storage/innobase/include/dict0crea.h | 16 ------ storage/innobase/include/dict0dict.h | 18 ------ storage/innobase/include/dict0dict.ic | 40 ------------- storage/innobase/include/fil0fil.h | 25 ++++----- storage/innobase/include/ibuf0ibuf.h | 14 ++--- storage/innobase/include/page0page.h | 4 +- storage/innobase/page/page0page.cc | 4 +- storage/innobase/row/row0quiesce.cc | 15 +++-- 13 files changed, 40 insertions(+), 211 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index d9af896e5c0..aa681e67201 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -6207,8 +6207,6 @@ database_corrupted: recv_recover_page(bpage); } - /* If space is being truncated then avoid ibuf operation. - During re-init we have already freed ibuf entries. */ if (uncompressed && !recv_no_ibuf_operations && (bpage->id.space() == 0 diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index c47e2f457fa..24662eed8af 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -957,87 +957,6 @@ dict_drop_index_tree( return false; } -/*******************************************************************//** -Recreate the index tree associated with a row in SYS_INDEXES table. -@return new root page number, or FIL_NULL on failure */ -ulint -dict_recreate_index_tree( -/*=====================*/ - const dict_table_t* - table, /*!< in/out: the table the index belongs to */ - btr_pcur_t* pcur, /*!< in/out: persistent cursor pointing to - record in the clustered index of - SYS_INDEXES table. The cursor may be - repositioned in this call. */ - mtr_t* mtr) /*!< in/out: mtr having the latch - on the record page. */ -{ - ut_ad(mutex_own(&dict_sys.mutex)); - ut_a(!dict_table_is_comp(dict_sys.sys_indexes)); - ut_ad(!table->space || table->space->id == table->space_id); - - ulint len; - const rec_t* rec = btr_pcur_get_rec(pcur); - - const byte* ptr = rec_get_nth_field_old( - rec, DICT_FLD__SYS_INDEXES__PAGE_NO, &len); - - ut_ad(len == 4); - - ut_ad(table->space_id == mach_read_from_4( - rec_get_nth_field_old(rec, DICT_FLD__SYS_INDEXES__SPACE, - &len))); - ut_ad(len == 4); - - if (!table->space) { - /* It is a single table tablespae and the .ibd file is - missing: do nothing. */ - - ib::warn() - << "Trying to TRUNCATE a missing .ibd file of table " - << table->name << "!"; - - return(FIL_NULL); - } - - ptr = rec_get_nth_field_old(rec, DICT_FLD__SYS_INDEXES__TYPE, &len); - ut_ad(len == 4); - ulint type = mach_read_from_4(ptr); - - ptr = rec_get_nth_field_old(rec, DICT_FLD__SYS_INDEXES__ID, &len); - ut_ad(len == 8); - index_id_t index_id = mach_read_from_8(ptr); - - /* We will need to commit the mini-transaction in order to avoid - deadlocks in the btr_create() call, because otherwise we would - be freeing and allocating pages in the same mini-transaction. */ - btr_pcur_store_position(pcur, mtr); - mtr_commit(mtr); - - mtr_start(mtr); - mtr->set_named_space(table->space); - btr_pcur_restore_position(BTR_MODIFY_LEAF, pcur, mtr); - - /* Find the index corresponding to this SYS_INDEXES record. */ - for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); - index != NULL; - index = UT_LIST_GET_NEXT(indexes, index)) { - if (index->id == index_id) { - ulint root_page_no = (index->type & DICT_FTS) - ? FIL_NULL - : btr_create(type, table->space, - index_id, index, mtr); - index->page = unsigned(root_page_no); - return root_page_no; - } - } - - ib::error() << "Failed to create index with index id " << index_id - << " of table " << table->name; - - return(FIL_NULL); -} - /*********************************************************************//** Creates a table create graph. @return own: table create node */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index bc032ed6aa0..b13e1397522 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1873,10 +1873,8 @@ for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces @return the tablespace -@retval NULL if missing or being deleted or truncated */ -UNIV_INTERN -fil_space_t* -fil_space_acquire_low(ulint id, bool silent) +@retval NULL if missing or being deleted */ +fil_space_t* fil_space_acquire_low(ulint id, bool silent) { fil_space_t* space; @@ -2203,9 +2201,7 @@ enum fil_operation_t { @param[in] space tablespace @param[in] count number of attempts so far @return 0 if no operations else count + 1. */ -static -ulint -fil_check_pending_ops(const fil_space_t* space, ulint count) +static ulint fil_check_pending_ops(const fil_space_t* space, ulint count) { ut_ad(mutex_own(&fil_system.mutex)); @@ -2216,7 +2212,7 @@ fil_check_pending_ops(const fil_space_t* space, ulint count) if (ulint n_pending_ops = space->n_pending_ops) { if (count > 5000) { - ib::warn() << "Trying to close/delete/truncate" + ib::warn() << "Trying to delete" " tablespace '" << space->name << "' but there are " << n_pending_ops << " pending operations on it."; @@ -2263,7 +2259,7 @@ fil_check_pending_io( ut_a(!(*node)->being_extended); if (count > 1000) { - ib::warn() << "Trying to delete/close/truncate" + ib::warn() << "Trying to delete" " tablespace '" << space->name << "' but there are " << space->n_pending_flushes diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 4869160b883..4deacd1c3b6 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -931,8 +931,9 @@ RemoteDatafile::create_link_file( prev_filepath = read_link_file(link_filepath); if (prev_filepath) { - /* Truncate will call this with an existing - link file which contains the same filepath. */ + /* Truncate (starting with MySQL 5.6, probably no + longer since MariaDB Server 10.2.19) used to call this + with an existing link file which contains the same filepath. */ bool same = !strcmp(prev_filepath, filepath); ut_free(prev_filepath); if (same) { diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 63a8e7e4bd9..c21a26a898d 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -4614,15 +4614,10 @@ reset_bit: ibuf_add_ops(ibuf->n_discarded_ops, dops); } -/*********************************************************************//** -Deletes all entries in the insert buffer for a given space id. This is used -in DISCARD TABLESPACE, IMPORT TABLESPACE, and 5.7 TRUNCATE TABLE recovery. -NOTE: this does not update the page free bitmaps in the space. The space will -become CORRUPT when you call this function! */ -void -ibuf_delete_for_discarded_space( -/*============================*/ - ulint space) /*!< in: space id */ +/** Delete all change buffer entries for a tablespace, +in DISCARD TABLESPACE, IMPORT TABLESPACE, or crash recovery. +@param[in] space missing or to-be-discarded tablespace */ +void ibuf_delete_for_discarded_space(ulint space) { mem_heap_t* heap; btr_pcur_t pcur; diff --git a/storage/innobase/include/dict0crea.h b/storage/innobase/include/dict0crea.h index 6039cb2c492..ed68b7f57b8 100644 --- a/storage/innobase/include/dict0crea.h +++ b/storage/innobase/include/dict0crea.h @@ -96,22 +96,6 @@ dict_create_index_tree( dict_index_t* index, /*!< in/out: index */ const trx_t* trx); /*!< in: InnoDB transaction handle */ -/*******************************************************************//** -Recreate the index tree associated with a row in SYS_INDEXES table. -@return new root page number, or FIL_NULL on failure */ -ulint -dict_recreate_index_tree( -/*======================*/ - const dict_table_t* table, /*!< in: the table the index - belongs to */ - btr_pcur_t* pcur, /*!< in/out: persistent cursor pointing - to record in the clustered index of - SYS_INDEXES table. The cursor may be - repositioned in this call. */ - mtr_t* mtr); /*!< in: mtr having the latch - on the record page. The mtr may be - committed and restarted in this call. */ - /** Drop the index tree associated with a row in SYS_INDEXES table. @param[in,out] rec SYS_INDEXES record @param[in,out] pcur persistent cursor on rec diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 19e8c25a3b8..901a9ad415a 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -910,24 +910,6 @@ inline ulint dict_table_extent_size(const dict_table_t* table) return FSP_EXTENT_SIZE; } -/*********************************************************************//** -Obtain exclusive locks on all index trees of the table. This is to prevent -accessing index trees while InnoDB is updating internal metadata for -operations such as truncate tables. */ -UNIV_INLINE -void -dict_table_x_lock_indexes( -/*======================*/ - dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((nonnull)); -/*********************************************************************//** -Release the exclusive locks on all index tree. */ -UNIV_INLINE -void -dict_table_x_unlock_indexes( -/*========================*/ - dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((nonnull)); /********************************************************************//** Checks if a column is in the ordering columns of the clustered index of a table. Column prefixes are treated like whole columns. diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 41c5c2220a4..b6d15f28a69 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -705,28 +705,6 @@ dict_tf_to_sys_tables_type( return(type); } -/*********************************************************************//** -Obtain exclusive locks on all index trees of the table. This is to prevent -accessing index trees while InnoDB is updating internal metadata for -operations such as truncate tables. */ -UNIV_INLINE -void -dict_table_x_lock_indexes( -/*======================*/ - dict_table_t* table) /*!< in: table */ -{ - dict_index_t* index; - - ut_ad(mutex_own(&dict_sys.mutex)); - - /* Loop through each index of the table and lock them */ - for (index = dict_table_get_first_index(table); - index != NULL; - index = dict_table_get_next_index(index)) { - rw_lock_x_lock(dict_index_get_lock(index)); - } -} - /*********************************************************************//** Returns true if the particular FTS index in the table is still syncing in the background, false otherwise. @@ -748,24 +726,6 @@ dict_fts_index_syncing( } return(false); } -/*********************************************************************//** -Release the exclusive locks on all index tree. */ -UNIV_INLINE -void -dict_table_x_unlock_indexes( -/*========================*/ - dict_table_t* table) /*!< in: table */ -{ - dict_index_t* index; - - ut_ad(mutex_own(&dict_sys.mutex)); - - for (index = dict_table_get_first_index(table); - index != NULL; - index = dict_table_get_next_index(index)) { - rw_lock_x_unlock(dict_index_get_lock(index)); - } -} /********************************************************************//** Gets the number of fields in the internal representation of an index, diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index a876076999d..a1efd28f7ed 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -92,18 +92,15 @@ struct fil_space_t { /** Log sequence number of the latest MLOG_INDEX_LOAD record that was found while parsing the redo log */ lsn_t enable_lsn; + /** set when an .ibd file is about to be deleted, + or an undo tablespace is about to be truncated. + When this is set following new ops are not allowed: + * read IO request + * ibuf merge + * file flush + Note that we can still possibly have new write operations + because we don't check this flag when doing flush batches. */ bool stop_new_ops; - /*!< we set this true when we start - deleting a single-table tablespace. - When this is set following new ops - are not allowed: - * read IO request - * ibuf merge - * file flush - Note that we can still possibly have - new write operations because we don't - check this flag when doing flush - batches. */ /** whether undo tablespace truncation is in progress */ bool is_being_truncated; #ifdef UNIV_DEBUG @@ -1112,10 +1109,8 @@ for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces @return the tablespace -@retval NULL if missing or being deleted or truncated */ -UNIV_INTERN -fil_space_t* -fil_space_acquire_low(ulint id, bool silent) +@retval NULL if missing or being deleted */ +fil_space_t* fil_space_acquire_low(ulint id, bool silent) MY_ATTRIBUTE((warn_unused_result)); /** Acquire a tablespace when it could be dropped concurrently. diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index 582ef2fda1b..4fbf5b6f70d 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -337,15 +337,11 @@ ibuf_merge_or_delete_for_page( ulint zip_size, bool update_ibuf_bitmap); -/*********************************************************************//** -Deletes all entries in the insert buffer for a given space id. This is used -in DISCARD TABLESPACE and IMPORT TABLESPACE. -NOTE: this does not update the page free bitmaps in the space. The space will -become CORRUPT when you call this function! */ -void -ibuf_delete_for_discarded_space( -/*============================*/ - ulint space); /*!< in: space id */ +/** Delete all change buffer entries for a tablespace, +in DISCARD TABLESPACE, IMPORT TABLESPACE, or crash recovery. +@param[in] space missing or to-be-discarded tablespace */ +void ibuf_delete_for_discarded_space(ulint space); + /** Contract the change buffer by reading pages to the buffer pool. @param[in] full If true, do a full contraction based on PCT_IO(100). If false, the size of contract batch is determined diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index cce4a92ae8b..a6dd19cc147 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -1023,9 +1023,7 @@ page_create_zip( buf_block_t* block, /*!< in/out: a buffer frame where the page is created */ dict_index_t* index, /*!< in: the index of the - page, or NULL when applying - TRUNCATE log - record during recovery */ + page */ ulint level, /*!< in: the B-tree level of the page */ trx_id_t max_trx_id, /*!< in: PAGE_MAX_TRX_ID */ diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index 2a345d87686..fa73b86f03f 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -448,9 +448,7 @@ page_create_zip( buf_block_t* block, /*!< in/out: a buffer frame where the page is created */ dict_index_t* index, /*!< in: the index of the - page, or NULL when applying - TRUNCATE log - record during recovery */ + page */ ulint level, /*!< in: the B-tree level of the page */ trx_id_t max_trx_id, /*!< in: PAGE_MAX_TRX_ID */ diff --git a/storage/innobase/row/row0quiesce.cc b/storage/innobase/row/row0quiesce.cc index 192545192d4..02c8c495e88 100644 --- a/storage/innobase/row/row0quiesce.cc +++ b/storage/innobase/row/row0quiesce.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2018, MariaDB Corporation. +Copyright (c) 2017, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -669,8 +669,11 @@ row_quiesce_set_state( } row_mysql_lock_data_dictionary(trx); - - dict_table_x_lock_indexes(table); + for (dict_index_t* index = dict_table_get_first_index(table); + index != NULL; + index = dict_table_get_next_index(index)) { + rw_lock_x_lock(&index->lock); + } switch (state) { case QUIESCE_START: @@ -687,7 +690,11 @@ row_quiesce_set_state( table->quiesce = state; - dict_table_x_unlock_indexes(table); + for (dict_index_t* index = dict_table_get_first_index(table); + index != NULL; + index = dict_table_get_next_index(index)) { + rw_lock_x_unlock(&index->lock); + } row_mysql_unlock_data_dictionary(trx); From 23d675453f182becb3bc5811bfa7e2e310cbed1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 12:44:04 +0300 Subject: [PATCH 3/7] MDEV-20675 Crash in SHOW ENGINE INNODB STATUS with innodb_force_recovery=5 Add a test case. MariaDB Server 10.2 is not affected. --- mysql-test/suite/innodb/t/innodb_force_recovery.test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mysql-test/suite/innodb/t/innodb_force_recovery.test b/mysql-test/suite/innodb/t/innodb_force_recovery.test index b5b48281161..9d7cf9d80b7 100644 --- a/mysql-test/suite/innodb/t/innodb_force_recovery.test +++ b/mysql-test/suite/innodb/t/innodb_force_recovery.test @@ -19,6 +19,7 @@ SET GLOBAL innodb_fast_shutdown = 0; --echo # Restart the server with innodb_force_recovery as 4. --let $restart_parameters= --innodb-force-recovery=4 --source include/restart_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; select * from t1; @@ -58,6 +59,7 @@ show tables; --echo # Restart the server with innodb_force_recovery as 5. --let $restart_parameters= --innodb-force-recovery=5 --source include/restart_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; select * from t2; @@ -98,6 +100,7 @@ show tables; --echo # Restart the server with innodb_force_recovery as 6. --let $restart_parameters= --innodb-force-recovery=6 --source include/restart_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; select * from t2; @@ -135,6 +138,7 @@ show tables; --echo # Restart the server with innodb_force_recovery=2 --let $restart_parameters= --innodb-force-recovery=2 --source include/restart_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; select * from t2; begin; @@ -152,6 +156,7 @@ connection default; --echo # Restart the server with innodb_force_recovery=3 --let $restart_parameters= --innodb-force-recovery=3 --source include/start_mysqld.inc +let $status=`SHOW ENGINE INNODB STATUS`; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; select * from t2; From 2d6719d7ee92843d5b0b9a27c7deaff5cacd4745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 12:48:55 +0300 Subject: [PATCH 4/7] MDEV-19514 preparation: Extend innodb.innodb-change-buffer-recovery Test innodb_read_only startup (which will be refused after a crash), and test also innodb_force_recovery=5, and extract some change buffer merge statistics. Omit any statistics about delete (purge) buffering, because purge could happen at any time. Use the sequence storage engine for populating the table. --- .../r/innodb-change-buffer-recovery.result | 36 +++++++++------- .../innodb-change-buffer-recovery-master.opt | 1 - .../t/innodb-change-buffer-recovery.test | 43 +++++++++++-------- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result b/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result index f03072053c3..156ef0e56be 100644 --- a/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result +++ b/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result @@ -2,6 +2,10 @@ # Bug#69122 - INNODB DOESN'T REDO-LOG INSERT BUFFER MERGE # OPERATION IF IT IS DONE IN-PLACE # +call mtr.add_suppression("InnoDB: innodb_read_only prevents crash recovery"); +call mtr.add_suppression("Plugin initialization aborted at srv0start\\.cc"); +call mtr.add_suppression("Plugin 'InnoDB'"); +FLUSH TABLES; CREATE TABLE t1( a INT AUTO_INCREMENT PRIMARY KEY, b CHAR(1), @@ -9,20 +13,7 @@ c INT, INDEX(b)) ENGINE=InnoDB STATS_PERSISTENT=0; SET GLOBAL innodb_change_buffering_debug = 1; -INSERT INTO t1 VALUES(0,'x',1); -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; +INSERT INTO t1 SELECT 0,'x',1 FROM seq_1_to_8192; BEGIN; SELECT b FROM t1 LIMIT 3; b @@ -38,8 +29,23 @@ SELECT b FROM t1 LIMIT 3; ERROR HY000: Lost connection to MySQL server during query disconnect con1; connection default; -FOUND 1 /Wrote log record for ibuf update in place operation/ in my_restart.err +FOUND 1 /Wrote log record for ibuf update in place operation/ in mysqld.1.err +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check Error Unknown storage engine 'InnoDB' +test.t1 check error Corrupt +FOUND 1 /innodb_read_only prevents crash recovery/ in mysqld.1.err +SELECT * FROM t1 LIMIT 1; +a b c +1 X 1 +SHOW ENGINE INNODB STATUS; +Type Name Status +InnoDB insert 0, delete mark 0 +SET GLOBAL innodb_fast_shutdown=0; CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK +SHOW ENGINE INNODB STATUS; +Type Name Status +InnoDB insert 79, delete mark 1 DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery-master.opt b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery-master.opt index 97b259ee047..e5d7090c883 100644 --- a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery-master.opt +++ b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery-master.opt @@ -1,2 +1 @@ ---log-error=$MYSQLTEST_VARDIR/tmp/my_restart.err --innodb_buffer_pool_size=24M diff --git a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test index d8cc21c550c..79d9cc814a0 100644 --- a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test +++ b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test @@ -11,6 +11,12 @@ --source include/not_valgrind.inc # This test is slow on buildbot. --source include/big_test.inc +--source include/have_sequence.inc + +call mtr.add_suppression("InnoDB: innodb_read_only prevents crash recovery"); +call mtr.add_suppression("Plugin initialization aborted at srv0start\\.cc"); +call mtr.add_suppression("Plugin 'InnoDB'"); +FLUSH TABLES; CREATE TABLE t1( a INT AUTO_INCREMENT PRIMARY KEY, @@ -27,25 +33,12 @@ ENGINE=InnoDB STATS_PERSISTENT=0; # change buffering is possible, so that the change buffer will be used # whenever possible. SET GLOBAL innodb_change_buffering_debug = 1; -let SEARCH_FILE = $MYSQLTEST_VARDIR/tmp/my_restart.err; +let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err; # Create enough rows for the table, so that the change buffer will be # used for modifying the secondary index page. There must be multiple # index pages, because changes to the root page are never buffered. -INSERT INTO t1 VALUES(0,'x',1); -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; -INSERT INTO t1 SELECT 0,b,c FROM t1; +INSERT INTO t1 SELECT 0,'x',1 FROM seq_1_to_8192; BEGIN; SELECT b FROM t1 LIMIT 3; @@ -63,10 +56,26 @@ SET DEBUG_DBUG='+d,crash_after_log_ibuf_upd_inplace'; SELECT b FROM t1 LIMIT 3; disconnect con1; connection default; - let SEARCH_PATTERN=Wrote log record for ibuf update in place operation; --source include/search_pattern_in_file.inc ---source include/start_mysqld.inc +--let $restart_parameters= --innodb-read-only +--source include/start_mysqld.inc CHECK TABLE t1; +--source include/shutdown_mysqld.inc +let SEARCH_PATTERN=innodb_read_only prevents crash recovery; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --innodb-force-recovery=5 +--source include/start_mysqld.inc +SELECT * FROM t1 LIMIT 1; +replace_regex /.*operations:.* (insert.*), delete \d.*discarded .*/\1/; +SHOW ENGINE INNODB STATUS; +# Slow shutdown will not merge the changes due to innodb_force_recovery=5. +SET GLOBAL innodb_fast_shutdown=0; +--let $restart_parameters= +--source include/restart_mysqld.inc +CHECK TABLE t1; +replace_regex /.*operations:.* (insert.*), delete \d.*discarded .*/\1/; +SHOW ENGINE INNODB STATUS; DROP TABLE t1; From 3e4931cdf3485a8762235401bd8ca3128ab31a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 13:18:22 +0300 Subject: [PATCH 5/7] MDEV-20675 Crash in SHOW ENGINE INNODB STATUS with innodb_force_recovery=5 lock_print_info::operator(): Do not dereference purge_sys.query in case it is NULL. We would not initialize purge_sys if innodb_force_recovery is set to 5 or 6. The test case will be added by merge from 10.2. --- storage/innobase/lock/lock0lock.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index f4a57e4955a..28055676f74 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4684,12 +4684,15 @@ lock_trx_print_locks( /** Functor to display all transactions */ struct lock_print_info { - lock_print_info(FILE* file, time_t now) : file(file), now(now) {} + lock_print_info(FILE* file, time_t now) : + file(file), now(now), + purge_trx(purge_sys.query ? purge_sys.query->trx : NULL) + {} void operator()(const trx_t* trx) const { ut_ad(mutex_own(&trx_sys.mutex)); - if (trx == purge_sys.query->trx) + if (UNIV_UNLIKELY(trx == purge_trx)) return; lock_trx_print_wait_and_mvcc_state(file, trx, now); @@ -4699,6 +4702,7 @@ struct lock_print_info FILE* const file; const time_t now; + const trx_t* const purge_trx; }; /*********************************************************************//** From 46facaedbffe8031f249fb7ecfe114273023fc66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 15:18:43 +0300 Subject: [PATCH 6/7] Fix GCC 9 -Wmaybe-uninitialized Always initialize ScopedStatementReplication::saved_binlog_format, so that GCC cannot emit a bogus warning about ScopedStatementReplication::~ScopedStatementReplication() using the variable. The code was originally introduced in commit d998da03069dca1aae83ec6af7eb407bbdc9fc58. --- sql/sql_class.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sql/sql_class.h b/sql/sql_class.h index 9bd486a540e..19f7265b24d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2016, Oracle and/or its affiliates. - Copyright (c) 2009, 2017, MariaDB Corporation. + Copyright (c) 2009, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -6761,11 +6761,12 @@ void dbug_serve_apcs(THD *thd, int n_calls); class ScopedStatementReplication { public: - ScopedStatementReplication(THD *thd) : thd(thd) - { - if (thd) - saved_binlog_format= thd->set_current_stmt_binlog_format_stmt(); - } + ScopedStatementReplication(THD *thd) : + saved_binlog_format(thd + ? thd->set_current_stmt_binlog_format_stmt() + : BINLOG_FORMAT_MIXED), + thd(thd) + {} ~ScopedStatementReplication() { if (thd) @@ -6773,8 +6774,8 @@ public: } private: - enum_binlog_format saved_binlog_format; - THD *thd; + const enum_binlog_format saved_binlog_format; + THD *const thd; }; #endif /* MYSQL_SERVER */ From 1f4ee3fa5a80e9715b0dc77b72fabb95a4045745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 26 Sep 2019 19:45:10 +0300 Subject: [PATCH 7/7] MDEV-20117 Assertion 0 failed in row_sel_get_clust_rec_for_mysql The crash scenario is as follows: (1) A non-empty table exists. (2) MDEV-15562 instant ADD/DROP/reorder has been invoked. (3) Some purgeable undo log exists for the table. (4) The table becomes empty, containing not even any delete-marked records, only containing the hidden metadata record that was added in (2). (5) An instant ADD/DROP/reorder column is executed, and the table is emptied and the (2) metadata removed. (6) Purge processes an undo log record from (3), which will refer to a non-existent clustered index field, because the metadata that was created in (2) was remoeved in (5). We fix this by adjusting step (5) so that we will never remove the MDEV-15562-style metadata record. Removing the MDEV-11369 metadata record (instant ADD COLUMN to the end of the table) is completely fine at any time when the table becomes empty, because dict_index_t::n_fields will remain unchanged. innobase_instant_try(): Never remove the MDEV-15562 metadata record. page_cur_delete_rec(): Do not reset FIL_PAGE_TYPE when the MDEV-15562 metadata record is being removed as part of btr_cur_pessimistic_update() invoked by innobase_instant_try(). --- .../suite/innodb/r/instant_alter_bugs.result | 12 +++++++++++ .../suite/innodb/t/instant_alter_bugs.test | 20 +++++++++++++++++++ storage/innobase/handler/handler0alter.cc | 3 ++- storage/innobase/page/page0cur.cc | 3 ++- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/instant_alter_bugs.result b/mysql-test/suite/innodb/r/instant_alter_bugs.result index 045b7468049..c630432585a 100644 --- a/mysql-test/suite/innodb/r/instant_alter_bugs.result +++ b/mysql-test/suite/innodb/r/instant_alter_bugs.result @@ -283,3 +283,15 @@ ALTER TABLE t CHANGE COLUMN alpha a INT WITHOUT SYSTEM VERSIONING, ALGORITHM=INSTANT; DROP TABLE t; set @@system_versioning_alter_history = error; +# +# MDEV-20117 Assertion 0 failed in row_sel_get_clust_rec_for_mysql +# +CREATE TABLE t (b INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t SET b=1; +ALTER TABLE t ADD COLUMN a INT FIRST, ALGORITHM=INSTANT; +DELETE FROM t; +ALTER TABLE t ADD COLUMN c INT, ALGORITHM=INSTANT; +ALTER TABLE t DROP COLUMN c, ALGORITHM=INSTANT; +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/innodb/t/instant_alter_bugs.test b/mysql-test/suite/innodb/t/instant_alter_bugs.test index fda8e88d70c..338d6972c43 100644 --- a/mysql-test/suite/innodb/t/instant_alter_bugs.test +++ b/mysql-test/suite/innodb/t/instant_alter_bugs.test @@ -293,3 +293,23 @@ ALTER TABLE t CHANGE COLUMN alpha a INT WITHOUT SYSTEM VERSIONING, ALGORITHM=INSTANT; DROP TABLE t; set @@system_versioning_alter_history = error; + +--echo # +--echo # MDEV-20117 Assertion 0 failed in row_sel_get_clust_rec_for_mysql +--echo # + +# This is not repeating the bug itself, but demonstrating that both +# parts of the fix are needed. +# To repeat the original bug, we should be somehow able to empty +# the table of user records while purgeable undo log records exist. +CREATE TABLE t (b INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t SET b=1; +ALTER TABLE t ADD COLUMN a INT FIRST, ALGORITHM=INSTANT; +DELETE FROM t; +ALTER TABLE t ADD COLUMN c INT, ALGORITHM=INSTANT; +# If page_cur_delete_rec() emptied the page (and wrongly reset the +# page type) during the previous ALTER TABLE, the following would hit +# an assertion failure because of root page type mismatch. +ALTER TABLE t DROP COLUMN c, ALGORITHM=INSTANT; +SELECT * FROM t; +DROP TABLE t; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 14125272daf..ab3ec7ba249 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -5820,7 +5820,8 @@ add_all_virtual: dberr_t err = DB_SUCCESS; if (rec_is_metadata(rec, *index)) { ut_ad(page_rec_is_user_rec(rec)); - if (!page_has_next(block->frame) + if (!rec_is_alter_metadata(rec, *index) + && !page_has_next(block->frame) && page_rec_is_last(rec, block->frame)) { goto empty_table; } diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index ded90c1c4f8..c2bdaee8c13 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -2327,7 +2327,8 @@ page_cur_delete_rec( /* The record must not be the supremum or infimum record. */ ut_ad(page_rec_is_user_rec(current_rec)); - if (page_get_n_recs(page) == 1 && !recv_recovery_is_on()) { + if (page_get_n_recs(page) == 1 && !recv_recovery_is_on() + && !rec_is_alter_metadata(current_rec, *index)) { /* Empty the page, unless we are applying the redo log during crash recovery. During normal operation, the page_create_empty() gets logged as one of MLOG_PAGE_CREATE,