From b9e09453971eb3e611ca9d087bc6198a0f2080be Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 4 Jan 2018 16:52:15 +0000 Subject: [PATCH 01/82] update libmariadb --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index fe129ed39f3..bc1777f8ce8 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit fe129ed39f33ba2b430aac91473baee84de88b12 +Subproject commit bc1777f8ce8cb5fc3b4f0e184d56871d5b809c20 From 0de565a56475e355eb77225981256348c698b9b6 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 4 Jan 2018 23:40:37 -0800 Subject: [PATCH 02/82] Fixed mdev-14852 Fails to reopen temp table within standard CTE If the specification of a CTE contains a reference to a temporary table then THD::open_temporary_table() must be called for this reference for any occurrence of the CTE in the query. By mistake this was done only for the first occurrences of CTEs. The patch fixes this problem in With_element::clone_parsed_spec(). It also moves there the call of check_dependencies_in_with_clauses() to its proper place before the call of check_table_access(). Additionally the patch optimizes the number of calls of the function check_dependencies_in_with_clauses(). --- mysql-test/r/cte_nonrecursive.result | 30 ++++++++++++++++++++++++++++ mysql-test/t/cte_nonrecursive.test | 29 +++++++++++++++++++++++++++ sql/sql_cte.cc | 7 +++++++ sql/sql_parse.cc | 22 +++++--------------- sql/sql_prepare.cc | 11 +++------- 5 files changed, 74 insertions(+), 25 deletions(-) diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index b1b41b1c5e7..c1f4c9fd486 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1265,3 +1265,33 @@ a 4 deallocate prepare stmt; drop table t1; +# +# MDEV-14852: CTE using temporary table in query +# with two references to the CTE +# +create temporary table t1 (i int); +insert into t1 values (5),(4),(1),(2),(3); +with +c1 as (select i from t1), +c2 as (select i from c1 where c1.i=2) +select i from c1 where i > 3 union select i from c2; +i +5 +4 +2 +drop table t1; +create table t1 (term char(10)); +create temporary table t2 (term char(10)); +insert into t1 values ('TERM01'),('TERM02'),('TERM03'); +insert into t2 values ('TERM02'),('TERM03'),('TERM04'); +with c1 as (select * from t1), c2 as (select * from t2) +(select * from c1 left outer join c2 on c1.term = c2.term) +union all +(select * from c1 right outer join c2 on c1.term = c2.term +where c1.term is null); +term term +TERM02 TERM02 +TERM03 TERM03 +TERM01 NULL +NULL TERM04 +drop table t1,t2; diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index 1f21dbcd36d..9436665bfee 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -853,3 +853,32 @@ execute stmt; deallocate prepare stmt; drop table t1; + +--echo # +--echo # MDEV-14852: CTE using temporary table in query +--echo # with two references to the CTE +--echo # + +create temporary table t1 (i int); +insert into t1 values (5),(4),(1),(2),(3); + +with +c1 as (select i from t1), +c2 as (select i from c1 where c1.i=2) +select i from c1 where i > 3 union select i from c2; + +drop table t1; + +create table t1 (term char(10)); +create temporary table t2 (term char(10)); + +insert into t1 values ('TERM01'),('TERM02'),('TERM03'); +insert into t2 values ('TERM02'),('TERM03'),('TERM04'); + +with c1 as (select * from t1), c2 as (select * from t2) +(select * from c1 left outer join c2 on c1.term = c2.term) +union all +(select * from c1 right outer join c2 on c1.term = c2.term + where c1.term is null); + +drop table t1,t2; diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index 601c1928d58..d12948fb624 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -817,12 +817,19 @@ st_select_lex_unit *With_element::clone_parsed_spec(THD *thd, parse_status= parse_sql(thd, &parser_state, 0); if (parse_status) goto err; + + if (check_dependencies_in_with_clauses(lex->with_clauses_list)) + goto err; + spec_tables= lex->query_tables; spec_tables_tail= 0; for (TABLE_LIST *tbl= spec_tables; tbl; tbl= tbl->next_global) { + if (!tbl->derived && !tbl->schema_table && + thd->open_temporary_table(tbl)) + goto err; spec_tables_tail= tbl; } if (check_table_access(thd, SELECT_ACL, spec_tables, FALSE, UINT_MAX, FALSE)) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index f00c74b155c..6d068eb99ac 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2990,6 +2990,9 @@ mysql_execute_command(THD *thd) thd->get_stmt_da()->opt_clear_warning_info(thd->query_id); } + if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) + DBUG_RETURN(1); + #ifdef HAVE_REPLICATION if (unlikely(thd->slave_thread)) { @@ -3446,14 +3449,6 @@ mysql_execute_command(THD *thd) ulong privileges_requested= lex->exchange ? SELECT_ACL | FILE_ACL : SELECT_ACL; - /* - The same function must be called for DML commands - when CTEs are supported in DML statements - */ - res= check_dependencies_in_with_clauses(thd->lex->with_clauses_list); - if (res) - break; - if (all_tables) res= check_table_access(thd, privileges_requested, @@ -3879,8 +3874,7 @@ mysql_execute_command(THD *thd) /* Copy temporarily the statement flags to thd for lock_table_names() */ uint save_thd_create_info_options= thd->lex->create_info.options; thd->lex->create_info.options|= create_info.options; - if (!(res= check_dependencies_in_with_clauses(lex->with_clauses_list))) - res= open_and_lock_tables(thd, create_info, lex->query_tables, TRUE, 0); + res= open_and_lock_tables(thd, create_info, lex->query_tables, TRUE, 0); thd->lex->create_info.options= save_thd_create_info_options; if (res) { @@ -4493,8 +4487,7 @@ end_with_restore_list: unit->set_limit(select_lex); - if (!(res= check_dependencies_in_with_clauses(lex->with_clauses_list)) && - !(res=open_and_lock_tables(thd, all_tables, TRUE, 0))) + if (!(res=open_and_lock_tables(thd, all_tables, TRUE, 0))) { MYSQL_INSERT_SELECT_START(thd->query()); /* @@ -4821,9 +4814,6 @@ end_with_restore_list: { List *lex_var_list= &lex->var_list; - if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) - goto error; - if ((check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE) || open_and_lock_tables(thd, all_tables, TRUE, 0))) goto error; @@ -6376,8 +6366,6 @@ static bool execute_sqlcom_select(THD *thd, TABLE_LIST *all_tables) new (thd->mem_root) Item_int(thd, (ulonglong) thd->variables.select_limit); } - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - return 1; if (!(res= open_and_lock_tables(thd, all_tables, TRUE, 0))) { diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index be78a76152e..390b70877f5 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -1573,8 +1573,6 @@ static int mysql_test_select(Prepared_statement *stmt, lex->select_lex.context.resolve_in_select_list= TRUE; ulong privilege= lex->exchange ? SELECT_ACL | FILE_ACL : SELECT_ACL; - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - goto error; if (tables) { if (check_table_access(thd, privilege, tables, FALSE, UINT_MAX, FALSE)) @@ -1841,9 +1839,6 @@ static bool mysql_test_create_table(Prepared_statement *stmt) if (create_table_precheck(thd, tables, create_table)) DBUG_RETURN(TRUE); - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - DBUG_RETURN(TRUE); - if (select_lex->item_list.elements) { /* Base table and temporary table are not in the same name space. */ @@ -2234,9 +2229,6 @@ static bool mysql_test_insert_select(Prepared_statement *stmt, if (insert_precheck(stmt->thd, tables)) return 1; - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - return 1; - /* store it, because mysql_insert_select_prepare_tester change it */ first_local_table= lex->select_lex.table_list.first; DBUG_ASSERT(first_local_table != 0); @@ -2339,6 +2331,9 @@ static bool check_prepared_statement(Prepared_statement *stmt) if (tables) thd->get_stmt_da()->opt_clear_warning_info(thd->query_id); + if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) + goto error; + if (sql_command_flags[sql_command] & CF_HA_CLOSE) mysql_ha_rm_tables(thd, tables); From 578345305e261d635bfc938a5dc2a22b839c2148 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 5 Jan 2018 10:17:29 -0800 Subject: [PATCH 03/82] Added a test case for mdev-13454: Improper error in ONLY_FULL_GROUP_BY sql_mode with condition_pushdown_for_derived=on This bug is a consequence of the bug mdev-14368 fixed in 5.5.59. --- mysql-test/r/derived_cond_pushdown.result | 64 +++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 25 +++++++++ 2 files changed, 89 insertions(+) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 42662b84a31..c588a953a2c 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -8807,3 +8807,67 @@ EXPLAIN } } drop table t1; +# +# MDEV-13454: consequence of mdev-14368 fixed for 5.5 +# +SET sql_mode = 'ONLY_FULL_GROUP_BY'; +create table t1 (id int, id2 int); +insert into t1 values (1,1),(2,3),(3,4),(7,2); +create table t2(id2 int); +insert t2 values (1),(2),(3); +SELECT * FROM t1 +LEFT OUTER JOIN +(SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); +id2 id ct +1 1 1 +3 2 1 +2 7 1 +EXPLAIN FORMAT=JSON SELECT * FROM t1 +LEFT OUTER JOIN +(SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "vc.ct > 0", + "materialized": { + "query_block": { + "select_id": 2, + "having_condition": "ct > 0", + "filesort": { + "sort_key": "t2.id2", + "temporary_table": { + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + }, + "block-nl-join": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 4, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "t1.id2 = vc.id2" + } + } +} +DROP TABLE t1,t2; +SET sql_mode = DEFAULT; diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index 9eb332b72d1..4ea6214075b 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -1565,3 +1565,28 @@ eval $q; eval explain format=json $q; drop table t1; + +--echo # +--echo # MDEV-13454: consequence of mdev-14368 fixed for 5.5 +--echo # + +SET sql_mode = 'ONLY_FULL_GROUP_BY'; + +create table t1 (id int, id2 int); +insert into t1 values (1,1),(2,3),(3,4),(7,2); + +create table t2(id2 int); +insert t2 values (1),(2),(3); + +let $q= +SELECT * FROM t1 + LEFT OUTER JOIN + (SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); + +eval $q; +eval EXPLAIN FORMAT=JSON $q; + +DROP TABLE t1,t2; + +SET sql_mode = DEFAULT; From 3a22d6c136ff5a03012102d9a852c5e16ac612d4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 5 Jan 2018 18:22:57 +0000 Subject: [PATCH 04/82] Fix conf_to_src build. 2cd316911309e3db4007aa224f7874bfbeb14032 broke conf_to_src, because strings library is now dependend on mysys (my_alloc etc are used now directly in string lib) Fix by adding appropriate dependency. Also exclude conf_to_src from VS IDE builds. EXCLUDE_FROM_ALL is not enough for that. --- strings/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/strings/CMakeLists.txt b/strings/CMakeLists.txt index 65eeb457f2d..2a7f0b71cc8 100644 --- a/strings/CMakeLists.txt +++ b/strings/CMakeLists.txt @@ -34,4 +34,5 @@ ADD_DEFINITIONS(-DDISABLE_MYSQL_THREAD_H) ADD_CONVENIENCE_LIBRARY(strings ${STRINGS_SOURCES}) ADD_EXECUTABLE(conf_to_src EXCLUDE_FROM_ALL conf_to_src.c) -TARGET_LINK_LIBRARIES(conf_to_src strings) +SET_TARGET_PROPERTIES(conf_to_src PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD TRUE) +TARGET_LINK_LIBRARIES(conf_to_src mysys strings) From 15b1840f43be8c660382458826e9d5a47d148a67 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 5 Jan 2018 12:13:23 -0800 Subject: [PATCH 05/82] Added the test case from for mdev-14777: Crash in MariaDB 10.2.12 on query using VIEW and WITH RECURSIVE. The cause of this crash was the same as of the crash reported in mdev-14755. --- mysql-test/r/cte_recursive.result | 28 +++++++++++++++++++++++++ mysql-test/t/cte_recursive.test | 34 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index ece5421e279..300539a75b2 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2944,3 +2944,31 @@ limit 1 ); ERROR HY000: Unacceptable mutual recursion with anchored table 'cte_1' drop table t1; +# +# mdev-14777: crash caused by the same as in mdev-14755 +# +CREATE TABLE t1 (i1 int NOT NULL, i2 int); +CREATE TABLE t2 (d1 int NOT NULL PRIMARY KEY); +CREATE TABLE t3 (i int ); +insert into t1 select seq,seq from seq_1_to_100000; +insert into t2 select seq from seq_1000_to_100000; +insert into t3 select seq from seq_1_to_1000; +SELECT * +FROM +( +SELECT * +FROM +( +WITH RECURSIVE rt AS +( +SELECT i2 P, i1 C FROM t1 WHERE i1 IN (SELECT d1 FROM t2) +UNION +SELECT t1.i2 P, rt.C C FROM t1, rt +) +SELECT C,P +FROM ( SELECT P,C FROM rt WHERE NOT EXISTS (SELECT 1 FROM t1) ) Y +) X +WHERE 1 = 1 +) K, t3; +C P i +drop table t1,t2,t3; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 928f8a4334b..4e9a1e6aa32 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1998,3 +1998,37 @@ set @var= ); drop table t1; + +--echo # +--echo # mdev-14777: crash caused by the same as in mdev-14755 +--echo # + +--source include/have_sequence.inc + +CREATE TABLE t1 (i1 int NOT NULL, i2 int); +CREATE TABLE t2 (d1 int NOT NULL PRIMARY KEY); +CREATE TABLE t3 (i int ); + +insert into t1 select seq,seq from seq_1_to_100000; +insert into t2 select seq from seq_1000_to_100000; +insert into t3 select seq from seq_1_to_1000; + +SELECT * +FROM +( + SELECT * + FROM + ( + WITH RECURSIVE rt AS + ( + SELECT i2 P, i1 C FROM t1 WHERE i1 IN (SELECT d1 FROM t2) + UNION + SELECT t1.i2 P, rt.C C FROM t1, rt + ) + SELECT C,P + FROM ( SELECT P,C FROM rt WHERE NOT EXISTS (SELECT 1 FROM t1) ) Y + ) X + WHERE 1 = 1 +) K, t3; + +drop table t1,t2,t3; From 73cf630ffc2e971fac68addc60c6fbf805665127 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Sat, 6 Jan 2018 23:51:37 +0530 Subject: [PATCH 06/82] Fix Compile Error while using Flag '-DUSE_ARIA_FOR_TMP_TABLES:BOOL=OFF' --- sql/sql_select.cc | 6 +++--- sql/sql_select.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 5d6945e866b..e80c02b52b4 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -17799,7 +17799,7 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, table->in_use->inc_status_created_tmp_tables(); table->in_use->query_plan_flags|= QPLAN_TMP_DISK; share->db_record_offset= 1; - table->created= TRUE; + table->set_created(); DBUG_RETURN(0); err: DBUG_RETURN(1); @@ -18299,8 +18299,8 @@ int rr_sequential_and_unpack(READ_RECORD *info) */ bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, - MARIA_COLUMNDEF *start_recinfo, - MARIA_COLUMNDEF **recinfo, + TMP_ENGINE_COLUMNDEF *start_recinfo, + TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options) { if (table->s->db_type() == TMP_ENGINE_HTON) diff --git a/sql/sql_select.h b/sql/sql_select.h index d4daa04f370..017f66a23b8 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2264,8 +2264,8 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options); bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, - MARIA_COLUMNDEF *start_recinfo, - MARIA_COLUMNDEF **recinfo, + TMP_ENGINE_COLUMNDEF *start_recinfo, + TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options); bool open_tmp_table(TABLE *table); void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps); From 16d308e21d91b880f1a59ab964a86e39643cdf1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 09:24:13 +0200 Subject: [PATCH 07/82] MDEV-14874 innodb_encrypt_log corrupts the log when the LSN crosses 32-bit boundary This bug affects both writing and reading encrypted redo log in MariaDB 10.1, starting from version 10.1.3 which added support for innodb_encrypt_log. That is, InnoDB crash recovery and Mariabackup will sometimes fail when innodb_encrypt_log is used. MariaDB 10.2 or Mariabackup 10.2 or later versions are not affected. log_block_get_start_lsn(): Remove. This function would cause trouble if a log segment that is being read is crossing a 32-bit boundary of the LSN, because this function does not allow the most significant 32 bits of the LSN to change. log_blocks_crypt(), log_encrypt_before_write(), log_decrypt_after_read(): Add the parameter "lsn" for the start LSN of the block. log_blocks_encrypt(): Remove (unused function). --- extra/mariabackup/xtrabackup.cc | 2 +- mysql-test/suite/mariabackup/huge_lsn.opt | 6 +++ mysql-test/suite/mariabackup/huge_lsn.result | 18 +++++++ mysql-test/suite/mariabackup/huge_lsn.test | 51 ++++++++++++++++++ storage/innobase/include/log0crypt.h | 6 ++- storage/innobase/log/log0crypt.cc | 54 ++++++-------------- storage/innobase/log/log0log.cc | 7 +-- storage/xtradb/include/log0crypt.h | 6 ++- storage/xtradb/log/log0crypt.cc | 54 ++++++-------------- storage/xtradb/log/log0log.cc | 7 +-- 10 files changed, 124 insertions(+), 87 deletions(-) create mode 100644 mysql-test/suite/mariabackup/huge_lsn.opt create mode 100644 mysql-test/suite/mariabackup/huge_lsn.result create mode 100644 mysql-test/suite/mariabackup/huge_lsn.test diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 437fc4aa7f9..38312086ec1 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2698,7 +2698,7 @@ xtrabackup_scan_log_recs( if (srv_encrypt_log) { log_encrypt_before_write(scanned_checkpoint_no, - log_sys->buf, write_size); + log_sys->buf, start_lsn, write_size); } if (ds_write(dst_log_file, log_sys->buf, write_size)) { diff --git a/mysql-test/suite/mariabackup/huge_lsn.opt b/mysql-test/suite/mariabackup/huge_lsn.opt new file mode 100644 index 00000000000..74a6450a1ef --- /dev/null +++ b/mysql-test/suite/mariabackup/huge_lsn.opt @@ -0,0 +1,6 @@ +--innodb-encrypt-log=ON +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filekey=FILE:$MTR_SUITE_DIR/filekeys-data.key +--loose-file-key-management-filename=$MTR_SUITE_DIR/filekeys-data.enc +--loose-file-key-management-encryption-algorithm=aes_cbc diff --git a/mysql-test/suite/mariabackup/huge_lsn.result b/mysql-test/suite/mariabackup/huge_lsn.result new file mode 100644 index 00000000000..740a436228c --- /dev/null +++ b/mysql-test/suite/mariabackup/huge_lsn.result @@ -0,0 +1,18 @@ +# +# MDEV-13416 mariabackup fails with EFAULT "Bad Address" +# +call mtr.add_suppression("InnoDB: New log files created"); +FOUND /InnoDB: .*started; log sequence number 17596481010700/ in mysqld.1.err +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup +INSERT INTO t VALUES(2); +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test new file mode 100644 index 00000000000..3a173d43a3c --- /dev/null +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -0,0 +1,51 @@ +--source include/not_embedded.inc + +--echo # +--echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" +--echo # + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; +call mtr.add_suppression("InnoDB: New log files created"); + +--source include/shutdown_mysqld.inc + +perl; +my $file= "$ENV{MYSQLD_DATADIR}/ibdata1"; +open(FILE, "+<", $file) or die "Unable to open $file\n"; +binmode FILE; +my $ps= $ENV{INNODB_PAGE_SIZE}; +my $page; +die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; +substr($page,26,8) = pack("NN", 4096, ~1024); +substr($page,0,4)=pack("N",0xdeadbeef); +substr($page,$ps-8,4)=pack("N",0xdeadbeef); +sysseek(FILE, 0, 0) || die "Unable to rewind $file\n"; +syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; +close(FILE) || die "Unable to close $file\n"; +EOF + +--remove_files_wildcard $MYSQLD_DATADIR ib_logfile* + +--source include/start_mysqld.inc +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +--let SEARCH_PATTERN= InnoDB: .*started; log sequence number 17596481010700 +--source include/search_pattern_in_file.inc + +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); + +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log +INSERT INTO t VALUES(2); +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +--source include/restart_and_restore.inc +--enable_result_log +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 0ad7e7d7037..b7a221e0a81 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -73,6 +73,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size); /*!< in: size of log blocks */ /******************************************************** @@ -83,6 +85,8 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size); /*!< in: log segment size */ /* Error codes for crypt info */ diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index 13d69118779..ec3c72cab43 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -70,22 +70,6 @@ struct crypt_info_t { static std::deque crypt_info; -/*********************************************************************//** -Get a log block's start lsn. -@return a log block's start lsn */ -static inline -lsn_t -log_block_get_start_lsn( -/*====================*/ - lsn_t lsn, /*!< in: checkpoint lsn */ - ulint log_block_no) /*!< in: log block number */ -{ - lsn_t start_lsn = - (lsn & (lsn_t)0xffffffff00000000ULL) | - (((log_block_no - 1) & (lsn_t)0x3fffffff) << 9); - return start_lsn; -} - /*********************************************************************//** Get crypt info from checkpoint. @return a crypt info or NULL if not present. */ @@ -162,6 +146,8 @@ Crypt_result log_blocks_crypt( /*=============*/ const byte* block, /*!< in: blocks before encrypt/decrypt*/ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ ulint size, /*!< in: size of block */ byte* dst_block, /*!< out: blocks after encrypt/decrypt */ int what, /*!< in: encrypt or decrypt*/ @@ -171,21 +157,18 @@ log_blocks_crypt( Crypt_result rc = MY_AES_OK; uint dst_len; byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; - byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; - lsn_t lsn = is_encrypt ? log_sys->lsn : srv_start_lsn; const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) { + for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE, + lsn += OS_FILE_LOG_BLOCK_SIZE) { ulint log_block_no = log_block_get_hdr_no(log_block); - lsn_t log_block_start_lsn = log_block_get_start_lsn( - lsn, log_block_no); const crypt_info_t* info = crypt_info == NULL ? get_crypt_info(log_block) : crypt_info; #ifdef DEBUG_CRYPT fprintf(stderr, "%s %lu chkpt: %lu key: %u lsn: %lu\n", - is_encrypt ? "crypt" : "decrypt", + what == ENCRYPTION_FLAG_ENCRYPT ? "crypt" : "decrypt", log_block_no, log_block_get_checkpoint_no(log_block), info ? info->key_version : 0, @@ -214,7 +197,7 @@ log_blocks_crypt( // (1-byte, only 5 bits are used). "+" means concatenate. bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); memcpy(aes_ctr_counter, info->crypt_nonce, 3); - mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn); + mach_write_to_8(aes_ctr_counter + 3, lsn); mach_write_to_4(aes_ctr_counter + 11, log_block_no); bzero(aes_ctr_counter + 15, 1); @@ -459,19 +442,6 @@ add_crypt_info( return true; } -/*********************************************************************//** -Encrypt log blocks. */ -UNIV_INTERN -Crypt_result -log_blocks_encrypt( -/*===============*/ - const byte* block, /*!< in: blocks before encryption */ - const ulint size, /*!< in: size of blocks, must be multiple of a log block */ - byte* dst_block) /*!< out: blocks after encryption */ -{ - return log_blocks_crypt(block, size, dst_block, ENCRYPTION_FLAG_ENCRYPT, NULL); -} - /*********************************************************************//** Set next checkpoint's key version to latest one, and generate current key. Key version 0 means no encryption. */ @@ -523,6 +493,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size) /*!< in: size of log blocks */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); @@ -541,7 +513,8 @@ log_encrypt_before_write( byte* dst_frame = (byte*)malloc(size); //encrypt log blocks content - Crypt_result result = log_blocks_crypt(block, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); + Crypt_result result = log_blocks_crypt( + block, lsn, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); if (result == MY_AES_OK) { ut_ad(block[0] == dst_frame[0]); @@ -561,13 +534,16 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size) /*!< in: log segment size */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); byte* dst_frame = (byte*)malloc(size); // decrypt log blocks content - Crypt_result result = log_blocks_crypt(frame, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); + Crypt_result result = log_blocks_crypt( + frame, lsn, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); if (result == MY_AES_OK) { memcpy(frame, dst_frame, size); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 1c9cbc3a219..7a242b76e85 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1380,7 +1380,7 @@ loop: ut_a(next_offset / UNIV_PAGE_SIZE <= ULINT_MAX); log_encrypt_before_write(log_sys->next_checkpoint_no, - buf, write_len); + buf, start_lsn, write_len); fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), @@ -2335,7 +2335,7 @@ loop: log_block_get_checksum(buf), source_offset); #endif - log_decrypt_after_read(buf, len); + log_decrypt_after_read(buf, start_lsn, len); #ifdef DEBUG_CRYPT fprintf(stderr, "AFTER DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx\n", @@ -2576,7 +2576,8 @@ loop: MONITOR_INC(MONITOR_LOG_IO); //TODO (jonaso): This must be dead code?? - log_encrypt_before_write(log_sys->next_checkpoint_no, buf, len); + log_encrypt_before_write(log_sys->next_checkpoint_no, + buf, start_lsn, len); fil_io(OS_FILE_WRITE | OS_FILE_LOG, false, group->archive_space_id, (ulint) (next_offset / UNIV_PAGE_SIZE), diff --git a/storage/xtradb/include/log0crypt.h b/storage/xtradb/include/log0crypt.h index 0ad7e7d7037..b7a221e0a81 100644 --- a/storage/xtradb/include/log0crypt.h +++ b/storage/xtradb/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -73,6 +73,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size); /*!< in: size of log blocks */ /******************************************************** @@ -83,6 +85,8 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size); /*!< in: log segment size */ /* Error codes for crypt info */ diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index a5fbbab17ef..2a0a7abb686 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -69,22 +69,6 @@ struct crypt_info_t { static std::deque crypt_info; -/*********************************************************************//** -Get a log block's start lsn. -@return a log block's start lsn */ -static inline -lsn_t -log_block_get_start_lsn( -/*====================*/ - lsn_t lsn, /*!< in: checkpoint lsn */ - ulint log_block_no) /*!< in: log block number */ -{ - lsn_t start_lsn = - (lsn & (lsn_t)0xffffffff00000000ULL) | - (((log_block_no - 1) & (lsn_t)0x3fffffff) << 9); - return start_lsn; -} - /*********************************************************************//** Get crypt info from checkpoint. @return a crypt info or NULL if not present. */ @@ -161,6 +145,8 @@ Crypt_result log_blocks_crypt( /*=============*/ const byte* block, /*!< in: blocks before encrypt/decrypt*/ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ ulint size, /*!< in: size of block */ byte* dst_block, /*!< out: blocks after encrypt/decrypt */ int what, /*!< in: encrypt or decrypt*/ @@ -170,21 +156,18 @@ log_blocks_crypt( Crypt_result rc = MY_AES_OK; uint dst_len; byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; - byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; - lsn_t lsn = is_encrypt ? log_sys->lsn : srv_start_lsn; const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) { + for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE, + lsn += OS_FILE_LOG_BLOCK_SIZE) { ulint log_block_no = log_block_get_hdr_no(log_block); - lsn_t log_block_start_lsn = log_block_get_start_lsn( - lsn, log_block_no); const crypt_info_t* info = crypt_info == NULL ? get_crypt_info(log_block) : crypt_info; #ifdef DEBUG_CRYPT fprintf(stderr, "%s %lu chkpt: %lu key: %u lsn: %lu\n", - is_encrypt ? "crypt" : "decrypt", + what == ENCRYPTION_FLAG_ENCRYPT ? "crypt" : "decrypt", log_block_no, log_block_get_checkpoint_no(log_block), info ? info->key_version : 0, @@ -213,7 +196,7 @@ log_blocks_crypt( // (1-byte, only 5 bits are used). "+" means concatenate. bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); memcpy(aes_ctr_counter, info->crypt_nonce, 3); - mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn); + mach_write_to_8(aes_ctr_counter + 3, lsn); mach_write_to_4(aes_ctr_counter + 11, log_block_no); bzero(aes_ctr_counter + 15, 1); @@ -458,19 +441,6 @@ add_crypt_info( return true; } -/*********************************************************************//** -Encrypt log blocks. */ -UNIV_INTERN -Crypt_result -log_blocks_encrypt( -/*===============*/ - const byte* block, /*!< in: blocks before encryption */ - const ulint size, /*!< in: size of blocks, must be multiple of a log block */ - byte* dst_block) /*!< out: blocks after encryption */ -{ - return log_blocks_crypt(block, size, dst_block, ENCRYPTION_FLAG_ENCRYPT, NULL); -} - /*********************************************************************//** Set next checkpoint's key version to latest one, and generate current key. Key version 0 means no encryption. */ @@ -522,6 +492,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size) /*!< in: size of log blocks */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); @@ -540,7 +512,8 @@ log_encrypt_before_write( byte* dst_frame = (byte*)malloc(size); //encrypt log blocks content - Crypt_result result = log_blocks_crypt(block, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); + Crypt_result result = log_blocks_crypt( + block, lsn, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); if (result == MY_AES_OK) { ut_ad(block[0] == dst_frame[0]); @@ -560,13 +533,16 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size) /*!< in: log segment size */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); byte* dst_frame = (byte*)malloc(size); // decrypt log blocks content - Crypt_result result = log_blocks_crypt(frame, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); + Crypt_result result = log_blocks_crypt( + frame, lsn, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); if (result == MY_AES_OK) { memcpy(frame, dst_frame, size); diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 1542f4a646f..8f8984f8880 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1490,7 +1490,7 @@ loop: ut_a(next_offset / UNIV_PAGE_SIZE <= ULINT_MAX); log_encrypt_before_write(log_sys->next_checkpoint_no, - buf, write_len); + buf, start_lsn, write_len); #ifdef DEBUG_CRYPT fprintf(stderr, "WRITE: block: %lu checkpoint: %lu %.8lx %.8lx\n", @@ -2582,7 +2582,7 @@ loop: log_block_get_checksum(buf), source_offset); #endif - log_decrypt_after_read(buf, len); + log_decrypt_after_read(buf, start_lsn, len); #ifdef DEBUG_CRYPT fprintf(stderr, "AFTER DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx\n", @@ -2890,7 +2890,8 @@ loop: MONITOR_INC(MONITOR_LOG_IO); //TODO (jonaso): This must be dead code?? - log_encrypt_before_write(log_sys->next_checkpoint_no, buf, len); + log_encrypt_before_write(log_sys->next_checkpoint_no, + buf, start_lsn, len); fil_io(OS_FILE_WRITE | OS_FILE_LOG, false, group->archive_space_id, 0, From ae7e1b9b13ddc23167bd058ff4718ac116d17126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 8 Jan 2018 12:25:31 +0200 Subject: [PATCH 08/82] MDEV-13262: innodb.deadlock_detect failed in buildbot There is a race condition on a test. con1 is the older transaction as it query started wait first. Test continues so that con1 gets lock wait timeout first. There is possibility that default connection gets lock timeout also or as con1 is rolled back it gets the locks it waited and does the update. Fixed by removing query outputs as they could vary and accepting success from default connection query. --- mysql-test/suite/innodb/r/deadlock_detect.result | 6 ------ mysql-test/suite/innodb/t/deadlock_detect.test | 14 +++++++++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/innodb/r/deadlock_detect.result b/mysql-test/suite/innodb/r/deadlock_detect.result index c3e3794ed21..4e14eff34b2 100644 --- a/mysql-test/suite/innodb/r/deadlock_detect.result +++ b/mysql-test/suite/innodb/r/deadlock_detect.result @@ -8,21 +8,15 @@ PRIMARY KEY(id) INSERT INTO t1 VALUES(1), (2), (3); BEGIN; SELECT * FROM t1 WHERE id = 1 FOR UPDATE; -id -1 connect con1,localhost,root,,; BEGIN; SELECT * FROM t1 WHERE id = 2 FOR UPDATE; -id -2 SELECT * FROM t1 WHERE id = 1 FOR UPDATE; connection default; SELECT * FROM t1 WHERE id = 2 FOR UPDATE; connection con1; -ERROR HY000: Lock wait timeout exceeded; try restarting transaction ROLLBACK; connection default; -ERROR HY000: Lock wait timeout exceeded; try restarting transaction ROLLBACK; DROP TABLE t1; disconnect con1; diff --git a/mysql-test/suite/innodb/t/deadlock_detect.test b/mysql-test/suite/innodb/t/deadlock_detect.test index 85d8c1f67f2..babdb54719f 100644 --- a/mysql-test/suite/innodb/t/deadlock_detect.test +++ b/mysql-test/suite/innodb/t/deadlock_detect.test @@ -18,6 +18,8 @@ CREATE TABLE t1( INSERT INTO t1 VALUES(1), (2), (3); +# We are not interested query results, only errors +--disable_result_log BEGIN; SELECT * FROM t1 WHERE id = 1 FOR UPDATE; @@ -39,12 +41,22 @@ reap; ROLLBACK; +# +# Note here that con1 is the older transaction as it +# query started wait first. Thus, con1 gets lock +# wait timeout first. There is possibility that +# default connection gets lock timeout also or +# as con1 is rolled back it gets the locks it waited +# and does the update. +# connection default; ---error ER_LOCK_WAIT_TIMEOUT +--error 0,ER_LOCK_WAIT_TIMEOUT reap; ROLLBACK; +--enable_result_log + DROP TABLE t1; disconnect con1; From 8099941b46dd5f7f79d361e6dcf378100305bd59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 10:58:36 +0200 Subject: [PATCH 09/82] MDEV-13487 Assertion failure in rec_get_trx_id() rec_get_trx_id(): Because rec is not necessarily residing in a buffer pool page (it could be an old version of a clustered index record, allocated from heap), remove the debug assertions that depend on page_align(rec). --- storage/innobase/rem/rem0rec.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index 37bf6e649a2..6cd8db5d37b 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2167,8 +2167,6 @@ rec_get_trx_id( const rec_t* rec, const dict_index_t* index) { - const page_t* page - = page_align(rec); ulint trx_id_col = dict_index_get_sys_col_pos(index, DATA_TRX_ID); const byte* trx_id; @@ -2179,11 +2177,7 @@ rec_get_trx_id( ulint* offsets = offsets_; ut_ad(trx_id_col <= MAX_REF_PARTS); - ut_ad(fil_page_index_page_check(page)); - ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) - == index->id); ut_ad(dict_index_is_clust(index)); - ut_ad(page_rec_is_leaf(rec)); ut_ad(trx_id_col > 0); ut_ad(trx_id_col != ULINT_UNDEFINED); From 899c5899becbaf18651215693fb455001476b577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 12:58:25 +0200 Subject: [PATCH 10/82] MLOG-13101 Debug assertion failed in recv_parse_or_apply_log_rec_body() recv_parse_or_apply_log_rec_body(): Tolerate MLOG_4BYTES for dummy-writing the FIL_PAGE_SPACE_ID, written by fil_crypt_rotate_page(). --- storage/innobase/log/log0recv.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 2e967a99121..6bb2806fb70 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1166,6 +1166,8 @@ parse_log: redo log been written with something older than InnoDB Plugin 1.0.4. */ ut_ad(0 + /* fil_crypt_rotate_page() writes this */ + || offs == FIL_PAGE_SPACE_ID || offs == IBUF_TREE_SEG_HEADER + IBUF_HEADER + FSEG_HDR_SPACE || offs == IBUF_TREE_SEG_HEADER From c903ba2f1ef5564d0eafd2803342f11cecb9bf01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 14:24:04 +0200 Subject: [PATCH 11/82] MDEV-13205 InnoDB: Failing assertion: !dict_index_is_online_ddl(index) upon ALTER TABLE dict_foreign_find_index(): Ignore incompletely created indexes. After a failed ADD UNIQUE INDEX, an incompletely created index could be left behind until the next ALTER TABLE statement. --- .../suite/innodb/r/innodb-index-online.result | 16 +++++++++++ .../suite/innodb/t/innodb-index-online.test | 27 +++++++++++++++++-- storage/innobase/dict/dict0dict.cc | 1 + storage/xtradb/dict/dict0dict.cc | 1 + 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-index-online.result b/mysql-test/suite/innodb/r/innodb-index-online.result index 77b7bc365aa..1dccc6ac4b8 100644 --- a/mysql-test/suite/innodb/r/innodb-index-online.result +++ b/mysql-test/suite/innodb/r/innodb-index-online.result @@ -339,6 +339,22 @@ ERROR 42000: Duplicate key name 'c2h' SET DEBUG_SYNC = 'RESET'; SET GLOBAL innodb_monitor_disable = module_ddl; DROP TABLE t1; +# +# MDEV-13205 assertion !dict_index_is_online_ddl(index) upon ALTER TABLE +# +CREATE TABLE t1 (c VARCHAR(64)) ENGINE=InnoDB; +SET DEBUG_SYNC = 'row_log_apply_before SIGNAL t1u_created WAIT_FOR dup_done'; +ALTER TABLE t1 ADD UNIQUE(c); +SET DEBUG_SYNC = 'now WAIT_FOR t1u_created'; +BEGIN; +INSERT INTO t1 VALUES('bar'),('bar'); +SET DEBUG_SYNC = 'now SIGNAL dup_done'; +ERROR 23000: Duplicate entry 'bar' for key 'c' +SET DEBUG_SYNC = 'RESET'; +CREATE TABLE t2 (c VARCHAR(64)) ENGINE=InnoDB; +ALTER TABLE t2 ADD FOREIGN KEY (c) REFERENCES t1 (c); +ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 150 "Foreign key constraint is incorrectly formed") +DROP TABLE t2,t1; SET GLOBAL innodb_file_per_table = @global_innodb_file_per_table_orig; SET GLOBAL innodb_monitor_enable = default; SET GLOBAL innodb_monitor_disable = default; diff --git a/mysql-test/suite/innodb/t/innodb-index-online.test b/mysql-test/suite/innodb/t/innodb-index-online.test index e09a65a878a..331541ea099 100644 --- a/mysql-test/suite/innodb/t/innodb-index-online.test +++ b/mysql-test/suite/innodb/t/innodb-index-online.test @@ -382,8 +382,6 @@ connection default; reap; --enable_parsing #remove below con1 disconnect if above test case is enabled -connection con1; -disconnect con1; connection default; SHOW CREATE TABLE t1; @@ -399,6 +397,31 @@ SET GLOBAL innodb_monitor_disable = module_ddl; DROP TABLE t1; +--echo # +--echo # MDEV-13205 assertion !dict_index_is_online_ddl(index) upon ALTER TABLE +--echo # +CREATE TABLE t1 (c VARCHAR(64)) ENGINE=InnoDB; +SET DEBUG_SYNC = 'row_log_apply_before SIGNAL t1u_created WAIT_FOR dup_done'; +send ALTER TABLE t1 ADD UNIQUE(c); + +connection con1; +SET DEBUG_SYNC = 'now WAIT_FOR t1u_created'; +BEGIN; +INSERT INTO t1 VALUES('bar'),('bar'); +SET DEBUG_SYNC = 'now SIGNAL dup_done'; + +connection default; +--error ER_DUP_ENTRY +reap; + +SET DEBUG_SYNC = 'RESET'; +disconnect con1; +CREATE TABLE t2 (c VARCHAR(64)) ENGINE=InnoDB; +--replace_regex /#sql-[0-9a-f_]*/#sql-temporary/ +--error ER_CANT_CREATE_TABLE +ALTER TABLE t2 ADD FOREIGN KEY (c) REFERENCES t1 (c); +DROP TABLE t2,t1; + # Check that all connections opened by test cases in this file are really # gone so execution of other tests won't be affected by their presence. --source include/wait_until_count_sessions.inc diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 11a4b892397..0c8a278586d 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -3364,6 +3364,7 @@ dict_foreign_find_index( if (types_idx != index && !(index->type & DICT_FTS) && !index->to_be_dropped + && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index a23b297e904..fce293976bf 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -3378,6 +3378,7 @@ dict_foreign_find_index( if (types_idx != index && !(index->type & DICT_FTS) && !index->to_be_dropped + && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, From 18ccbf014adbc9eff74407ef15b3b6872196201e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 17:09:21 +0200 Subject: [PATCH 12/82] Make the MDEV-14874 test case more robust --- mysql-test/suite/mariabackup/huge_lsn.result | 3 ++- mysql-test/suite/mariabackup/huge_lsn.test | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/mariabackup/huge_lsn.result b/mysql-test/suite/mariabackup/huge_lsn.result index 740a436228c..f2202c20968 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.result +++ b/mysql-test/suite/mariabackup/huge_lsn.result @@ -2,10 +2,11 @@ # MDEV-13416 mariabackup fails with EFAULT "Bad Address" # call mtr.add_suppression("InnoDB: New log files created"); -FOUND /InnoDB: .*started; log sequence number 17596481010700/ in mysqld.1.err +FOUND /InnoDB: New log files created, LSN=175964\d{8}/ in mysqld.1.err CREATE TABLE t(i INT) ENGINE INNODB; INSERT INTO t VALUES(1); # xtrabackup backup +SET GLOBAL innodb_flush_log_at_trx_commit=1; INSERT INTO t VALUES(2); # xtrabackup prepare # shutdown server diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 3a173d43a3c..37be33916e7 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -28,8 +28,9 @@ EOF --remove_files_wildcard $MYSQLD_DATADIR ib_logfile* --source include/start_mysqld.inc +let SEARCH_RANGE= -50000; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; ---let SEARCH_PATTERN= InnoDB: .*started; log sequence number 17596481010700 +--let SEARCH_PATTERN= InnoDB: New log files created, LSN=175964\d{8} --source include/search_pattern_in_file.inc CREATE TABLE t(i INT) ENGINE INNODB; @@ -40,6 +41,7 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; --enable_result_log +SET GLOBAL innodb_flush_log_at_trx_commit=1; INSERT INTO t VALUES(2); echo # xtrabackup prepare; --disable_result_log From 919169e1f959744e609380132d8a561b96659811 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 8 Jan 2018 17:01:55 +0000 Subject: [PATCH 13/82] Fix warning --- sql/handler.cc | 2 +- win/create_def_file.js | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/handler.cc b/sql/handler.cc index 93db0b09abf..10c2b65bbc0 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -2068,7 +2068,7 @@ uint get_sql_xid(XID *xid, char *buf) MY_INT64_NUM_DECIMAL_DIGITS, -10, xid->formatID); } - return buf - orig_buf; + return (uint)(buf - orig_buf); } diff --git a/win/create_def_file.js b/win/create_def_file.js index 25bbbb4eb3d..9a178510ca8 100644 --- a/win/create_def_file.js +++ b/win/create_def_file.js @@ -257,4 +257,9 @@ function addToResponseFile(filename, responseFile) responseFile.WriteLine("\""+fso.GetFile(filename).Path+"\""); } } + else + { + echo("file " + filename + " not found. Can't generate symbols file"); + WScript.Quit (1); + } } From 075f61a1d43c6805503735fc3dd118b90ae939de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 9 Jan 2018 11:30:36 +0200 Subject: [PATCH 14/82] Revert part of commit fec844aca88e1c6b9c36bb0b811e92d9d023ffb9 row_insert_for_mysql(): Remove some duplicated code --- storage/innobase/row/row0mysql.cc | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 335ca35db95..7db855244ab 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1424,17 +1424,6 @@ row_insert_for_mysql( } else if (high_level_read_only) { return(DB_READ_ONLY); } - DBUG_EXECUTE_IF("mark_table_corrupted", { - /* Mark the table corrupted for the clustered index */ - dict_index_t* index = dict_table_get_first_index(table); - ut_ad(dict_index_is_clust(index)); - dict_set_corrupted(index, trx, "INSERT TABLE"); }); - - if (dict_table_is_corrupted(table)) { - - ib::error() << "Table " << table->name << " is corrupt."; - return(DB_TABLE_CORRUPT); - } DBUG_EXECUTE_IF("mark_table_corrupted", { /* Mark the table corrupted for the clustered index */ From 07aa98597984391a6e8c85f634106aef4b096a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 9 Jan 2018 12:37:58 +0200 Subject: [PATCH 15/82] MDEV-14776: InnoDB Monitor output generated by specific error is flooding error logs innodb/buf_LRU_get_free_block Add debug instrumentation to produce error message about no free pages. Print error message only once and do not enable innodb monitor. xtradb/buf_LRU_get_free_block Add debug instrumentation to produce error message about no free pages. Print error message only once and do not enable innodb monitor. Remove code that does not seem to be used. innodb-lru-force-no-free-page.test New test case to force produce desired error message. --- .../r/innodb-lru-force-no-free-page.result | 9 ++ .../t/innodb-lru-force-no-free-page.test | 24 ++++ storage/innobase/buf/buf0lru.cc | 76 +++++----- storage/xtradb/buf/buf0lru.cc | 136 ++++++------------ 4 files changed, 107 insertions(+), 138 deletions(-) create mode 100644 mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result create mode 100644 mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test diff --git a/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result new file mode 100644 index 00000000000..d095f4e32e7 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result @@ -0,0 +1,9 @@ +call mtr.add_suppression("\\[Warning\\] InnoDB: Difficult to find free blocks in the buffer pool."); +SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; +CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; +BEGIN; +INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); +COMMIT; +SET SESSION debug_dbug=""; +DROP TABLE t1; +FOUND /InnoDB: Difficult to find free blocks / in mysqld.1.err diff --git a/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test new file mode 100644 index 00000000000..e358446eca9 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test @@ -0,0 +1,24 @@ +--source include/have_innodb.inc +--source include/have_debug.inc + +call mtr.add_suppression("\\[Warning\\] InnoDB: Difficult to find free blocks in the buffer pool."); + +SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; + +CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; +BEGIN; +INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); +COMMIT; + +SET SESSION debug_dbug=""; + +DROP TABLE t1; + +# +# There should be only one message +# +let SEARCH_RANGE= -50000; +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +--let SEARCH_PATTERN=InnoDB: Difficult to find free blocks +--source include/search_pattern_in_file.inc + diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index a047b28f4fd..b1b63e72a45 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -90,6 +90,10 @@ during LRU eviction. */ frames in the buffer pool, we set this to TRUE */ static ibool buf_lru_switched_on_innodb_mon = FALSE; +/** True if diagnostic message about difficult to find free blocks +in the buffer bool has already printed. */ +static bool buf_lru_free_blocks_error_printed; + /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -1045,8 +1049,6 @@ buf_LRU_get_free_block( ibool freed = FALSE; ulint n_iterations = 0; ulint flush_failures = 0; - ibool mon_value_was = FALSE; - ibool started_monitor = FALSE; MONITOR_INC(MONITOR_LRU_GET_FREE_SEARCH); loop: @@ -1054,6 +1056,11 @@ loop: buf_LRU_check_size_of_non_data_objects(buf_pool); + DBUG_EXECUTE_IF("ib_lru_force_no_free_page", + if (!buf_lru_free_blocks_error_printed) { + n_iterations = 21; + goto not_found;}); + /* If there is a block in the free list, take it */ block = buf_LRU_get_free_only(buf_pool); @@ -1062,16 +1069,11 @@ loop: buf_pool_mutex_exit(buf_pool); ut_ad(buf_pool_from_block(block) == buf_pool); memset(&block->page.zip, 0, sizeof block->page.zip); - - if (started_monitor) { - srv_print_innodb_monitor = - static_cast(mon_value_was); - } - return(block); } freed = FALSE; + if (buf_pool->try_LRU_scan || n_iterations > 0) { /* If no block was in the free list, search from the end of the LRU list and try to free a block there. @@ -1094,6 +1096,10 @@ loop: } } +#ifndef DBUG_OFF +not_found: +#endif + buf_pool_mutex_exit(buf_pool); if (freed) { @@ -1101,40 +1107,26 @@ loop: } - if (n_iterations > 20) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: difficult to find free blocks in\n" - "InnoDB: the buffer pool (%lu search iterations)!\n" - "InnoDB: %lu failed attempts to flush a page!" - " Consider\n" - "InnoDB: increasing the buffer pool size.\n" - "InnoDB: It is also possible that" - " in your Unix version\n" - "InnoDB: fsync is very slow, or" - " completely frozen inside\n" - "InnoDB: the OS kernel. Then upgrading to" - " a newer version\n" - "InnoDB: of your operating system may help." - " Look at the\n" - "InnoDB: number of fsyncs in diagnostic info below.\n" - "InnoDB: Pending flushes (fsync) log: %lu;" - " buffer pool: %lu\n" - "InnoDB: %lu OS file reads, %lu OS file writes," - " %lu OS fsyncs\n" - "InnoDB: Starting InnoDB Monitor to print further\n" - "InnoDB: diagnostics to the standard output.\n", - (ulong) n_iterations, - (ulong) flush_failures, - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); + if (n_iterations > 20 && !buf_lru_free_blocks_error_printed) { + ib_logf(IB_LOG_LEVEL_WARN, + "Difficult to find free blocks in" + " the buffer pool (" ULINTPF " search iterations)! " + ULINTPF " failed attempts to flush a page!", + n_iterations, flush_failures); + ib_logf(IB_LOG_LEVEL_INFO, + "Consider increasing the buffer pool size."); + ib_logf(IB_LOG_LEVEL_INFO, + "Pending flushes (fsync) log: " ULINTPF + " buffer pool: " ULINTPF + " OS file reads: " ULINTPF " OS file writes: " + ULINTPF " OS fsyncs: " ULINTPF "", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); - mon_value_was = srv_print_innodb_monitor; - started_monitor = TRUE; - srv_print_innodb_monitor = TRUE; - os_event_set(srv_monitor_event); + buf_lru_free_blocks_error_printed = true; } /* If we have scanned the whole LRU and still are unable to diff --git a/storage/xtradb/buf/buf0lru.cc b/storage/xtradb/buf/buf0lru.cc index c71d45009e4..a84f2a832f3 100644 --- a/storage/xtradb/buf/buf0lru.cc +++ b/storage/xtradb/buf/buf0lru.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -87,6 +87,10 @@ buffer pools. */ frames in the buffer pool, we set this to TRUE */ static ibool buf_lru_switched_on_innodb_mon = FALSE; +/** True if diagnostic message about difficult to find free blocks +in the buffer bool has already printed. */ +static bool buf_lru_free_blocks_error_printed; + /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -1080,68 +1084,39 @@ buf_LRU_check_size_of_non_data_objects( } /** Diagnose failure to get a free page and request InnoDB monitor output in -the error log if more than two seconds have been spent already. +the error log if it has not yet printed. @param[in] n_iterations how many buf_LRU_get_free_page iterations already completed -@param[in] started_ms timestamp in ms of when the attempt to get the - free page started @param[in] flush_failures how many times single-page flush, if allowed, has failed -@param[out] mon_value_was previous srv_print_innodb_monitor value -@param[out] started_monitor whether InnoDB monitor print has been requested */ static void -buf_LRU_handle_lack_of_free_blocks(ulint n_iterations, ulint started_ms, - ulint flush_failures, - ibool *mon_value_was, - ibool *started_monitor) +buf_LRU_handle_lack_of_free_blocks( + ulint n_iterations, + ulint flush_failures) { - static ulint last_printout_ms = 0; + if (n_iterations > 20 && !buf_lru_free_blocks_error_printed) { + ib_logf(IB_LOG_LEVEL_WARN, + "Difficult to find free blocks in" + " the buffer pool (" ULINTPF " search iterations)! " + ULINTPF " failed attempts to flush a page!", + n_iterations, flush_failures); + ib_logf(IB_LOG_LEVEL_INFO, + "Consider increasing the buffer pool size."); + ib_logf(IB_LOG_LEVEL_INFO, + "Pending flushes (fsync) log: " ULINTPF + " buffer pool: " ULINTPF + " OS file reads: " ULINTPF " OS file writes: " + ULINTPF " OS fsyncs: " ULINTPF "", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); - /* Legacy algorithm started warning after at least 2 seconds, we - emulate this. */ - const ulint current_ms = ut_time_ms(); - - if ((current_ms > started_ms + 2000) - && (current_ms > last_printout_ms + 2000)) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: difficult to find free blocks in\n" - "InnoDB: the buffer pool (%lu search iterations)!\n" - "InnoDB: %lu failed attempts to flush a page!" - " Consider\n" - "InnoDB: increasing the buffer pool size.\n" - "InnoDB: It is also possible that" - " in your Unix version\n" - "InnoDB: fsync is very slow, or" - " completely frozen inside\n" - "InnoDB: the OS kernel. Then upgrading to" - " a newer version\n" - "InnoDB: of your operating system may help." - " Look at the\n" - "InnoDB: number of fsyncs in diagnostic info below.\n" - "InnoDB: Pending flushes (fsync) log: %lu;" - " buffer pool: %lu\n" - "InnoDB: %lu OS file reads, %lu OS file writes," - " %lu OS fsyncs\n" - "InnoDB: Starting InnoDB Monitor to print further\n" - "InnoDB: diagnostics to the standard output.\n", - (ulong) n_iterations, - (ulong) flush_failures, - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - last_printout_ms = current_ms; - *mon_value_was = srv_print_innodb_monitor; - *started_monitor = TRUE; - srv_print_innodb_monitor = TRUE; - os_event_set(lock_sys->timeout_event); + buf_lru_free_blocks_error_printed = true; } - } /** The maximum allowed backoff sleep time duration, microseconds */ @@ -1189,9 +1164,6 @@ buf_LRU_get_free_block( ibool freed = FALSE; ulint n_iterations = 0; ulint flush_failures = 0; - ibool mon_value_was = FALSE; - ibool started_monitor = FALSE; - ulint started_ms = 0; ut_ad(!mutex_own(&buf_pool->LRU_list_mutex)); @@ -1199,42 +1171,20 @@ buf_LRU_get_free_block( loop: buf_LRU_check_size_of_non_data_objects(buf_pool); - /* If there is a block in the free list, take it */ - if (DBUG_EVALUATE_IF("simulate_lack_of_pages", true, false)) { + DBUG_EXECUTE_IF("ib_lru_force_no_free_page", + if (!buf_lru_free_blocks_error_printed) { + n_iterations = 21; + goto not_found;}); - block = NULL; - - if (srv_debug_monitor_printed) - DBUG_SET("-d,simulate_lack_of_pages"); - - } else if (DBUG_EVALUATE_IF("simulate_recovery_lack_of_pages", - recv_recovery_on, false)) { - - block = NULL; - - if (srv_debug_monitor_printed) - DBUG_SUICIDE(); - } else { - - block = buf_LRU_get_free_only(buf_pool); - } + block = buf_LRU_get_free_only(buf_pool); if (block) { ut_ad(buf_pool_from_block(block) == buf_pool); memset(&block->page.zip, 0, sizeof block->page.zip); - - if (started_monitor) { - srv_print_innodb_monitor = - static_cast(mon_value_was); - } - return(block); } - if (!started_ms) - started_ms = ut_time_ms(); - if (srv_empty_free_list_algorithm == SRV_EMPTY_FREE_LIST_BACKOFF && buf_lru_manager_is_active && (srv_shutdown_state == SRV_SHUTDOWN_NONE @@ -1272,10 +1222,7 @@ loop: : FREE_LIST_BACKOFF_LOW_PRIO_DIVIDER)); } - buf_LRU_handle_lack_of_free_blocks(n_iterations, started_ms, - flush_failures, - &mon_value_was, - &started_monitor); + buf_LRU_handle_lack_of_free_blocks(n_iterations, flush_failures); n_iterations++; @@ -1314,13 +1261,8 @@ loop: mutex_exit(&buf_pool->flush_state_mutex); - if (DBUG_EVALUATE_IF("simulate_recovery_lack_of_pages", true, false) - || DBUG_EVALUATE_IF("simulate_lack_of_pages", true, false)) { - - buf_pool->try_LRU_scan = false; - } - freed = FALSE; + if (buf_pool->try_LRU_scan || n_iterations > 0) { /* If no block was in the free list, search from the @@ -1345,9 +1287,11 @@ loop: } - buf_LRU_handle_lack_of_free_blocks(n_iterations, started_ms, - flush_failures, &mon_value_was, - &started_monitor); +#ifndef DBUG_OFF +not_found: +#endif + + buf_LRU_handle_lack_of_free_blocks(n_iterations, flush_failures); /* If we have scanned the whole LRU and still are unable to find a free block then we should sleep here to let the From 7d201d7b3000bc7cf4e9a4360ad55ed4c85c60d8 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sat, 6 Jan 2018 09:32:47 -0800 Subject: [PATCH 16/82] Fixed mdev-14879 Lost rows for query using recursive CTE with recursive reference in subquery If a recursive CTE uses a subquery with recursive reference then the virtual function reset() must be called after each iteration performed at the execution of the CTE. --- mysql-test/r/cte_recursive.result | 78 +++++++++++++++++++++++++++++++ mysql-test/t/cte_recursive.test | 53 +++++++++++++++++++++ sql/sql_union.cc | 1 + 3 files changed, 132 insertions(+) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index 300539a75b2..7bbdcb47f26 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2972,3 +2972,81 @@ WHERE 1 = 1 ) K, t3; C P i drop table t1,t2,t3; +# +# mdev-14879: subquery with recursive reference in WHERE of CTE +# +create table flights +(departure varchar(32), +arrival varchar(32), +carrier varchar(20), +flight_number char(7)); +insert into flights values +('Seattle', 'Frankfurt', 'Lufthansa', 'LH 491'), +('Seattle', 'Chicago', 'American', 'AA 2573'), +('Seattle', 'Los Angeles', 'Alaska Air', 'AS 410'), +('Chicago', 'New York', 'American', 'AA 375'), +('Chicago', 'Montreal', 'Air Canada', 'AC 3053'), +('Los Angeles', 'New York', 'Delta', 'DL 1197'), +('Moscow', 'Tokyo', 'Aeroflot', 'SU 264'), +('New York', 'Paris', 'Air France', 'AF 23'), +('Frankfurt', 'Moscow', 'Lufthansa', 'LH 1444'), +('Tokyo', 'Seattle', 'ANA', 'NH 178'), +('Los Angeles', 'Tokyo', 'ANA', 'NH 175'), +('Moscow', 'Los Angeles', 'Aeroflot', 'SU 106'), +('Montreal', 'Paris', 'Air Canada', 'AC 870'), +('Cairo', 'Paris', 'Air France', 'AF 503'), +('New York', 'Seattle', 'American', 'AA 45'), +('Paris', 'Chicago', 'Air France', 'AF 6734'); +with recursive destinations (city) as +( select a.arrival from flights a where a.departure='Cairo' + union +select b.arrival from destinations r, flights b where r.city=b.departure) +select * from destinations; +city +Paris +Chicago +New York +Montreal +Seattle +Frankfurt +Los Angeles +Moscow +Tokyo +set standard_compliant_cte=0; +with recursive destinations (city, legs) as +( +select a.arrival, 1 from flights a where a.departure='Cairo' + union +select b.arrival, r.legs + 1 from destinations r, flights b +where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; +city legs +Paris 1 +Chicago 2 +New York 3 +Montreal 3 +Seattle 4 +Frankfurt 5 +Los Angeles 5 +Moscow 6 +Tokyo 6 +explain extended with recursive destinations (city, legs) as +( +select a.arrival, 1 from flights a where a.departure='Cairo' + union +select b.arrival, r.legs + 1 from destinations r, flights b +where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 16 100.00 +2 DERIVED a ALL NULL NULL NULL NULL 16 100.00 Using where +3 RECURSIVE UNION b ALL NULL NULL NULL NULL 16 100.00 Using where +3 RECURSIVE UNION ref key0 key0 35 test.b.departure 2 100.00 +4 DEPENDENT SUBQUERY ALL NULL NULL NULL NULL 16 100.00 Using where +NULL UNION RESULT ALL NULL NULL NULL NULL NULL NULL +Warnings: +Note 1003 with recursive destinations as (select `test`.`a`.`arrival` AS `city`,1 AS `legs` from `test`.`flights` `a` where `test`.`a`.`departure` = 'Cairo' union select `test`.`b`.`arrival` AS `arrival`,`r`.`legs` + 1 AS `r.legs + 1` from `destinations` `r` join `test`.`flights` `b` where `r`.`city` = `test`.`b`.`departure` and !(`test`.`b`.`arrival`,(select `destinations`.`city` from `destinations` where trigcond(`test`.`b`.`arrival` = `destinations`.`city` or `destinations`.`city` is null) having trigcond(`destinations`.`city` is null))))select `destinations`.`city` AS `city`,`destinations`.`legs` AS `legs` from `destinations` +set standard_compliant_cte=default; +drop table flights; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 4e9a1e6aa32..332f6cfa49b 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -2032,3 +2032,56 @@ FROM ) K, t3; drop table t1,t2,t3; + +--echo # +--echo # mdev-14879: subquery with recursive reference in WHERE of CTE +--echo # + +create table flights +(departure varchar(32), + arrival varchar(32), + carrier varchar(20), + flight_number char(7)); + +insert into flights values +('Seattle', 'Frankfurt', 'Lufthansa', 'LH 491'), +('Seattle', 'Chicago', 'American', 'AA 2573'), +('Seattle', 'Los Angeles', 'Alaska Air', 'AS 410'), +('Chicago', 'New York', 'American', 'AA 375'), +('Chicago', 'Montreal', 'Air Canada', 'AC 3053'), +('Los Angeles', 'New York', 'Delta', 'DL 1197'), +('Moscow', 'Tokyo', 'Aeroflot', 'SU 264'), +('New York', 'Paris', 'Air France', 'AF 23'), +('Frankfurt', 'Moscow', 'Lufthansa', 'LH 1444'), +('Tokyo', 'Seattle', 'ANA', 'NH 178'), +('Los Angeles', 'Tokyo', 'ANA', 'NH 175'), +('Moscow', 'Los Angeles', 'Aeroflot', 'SU 106'), +('Montreal', 'Paris', 'Air Canada', 'AC 870'), +('Cairo', 'Paris', 'Air France', 'AF 503'), +('New York', 'Seattle', 'American', 'AA 45'), +('Paris', 'Chicago', 'Air France', 'AF 6734'); + +with recursive destinations (city) as +( select a.arrival from flights a where a.departure='Cairo' + union + select b.arrival from destinations r, flights b where r.city=b.departure) +select * from destinations; + +set standard_compliant_cte=0; + +let $q= +with recursive destinations (city, legs) as +( + select a.arrival, 1 from flights a where a.departure='Cairo' + union + select b.arrival, r.legs + 1 from destinations r, flights b + where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; + +eval $q; +eval explain extended $q; + +set standard_compliant_cte=default; + +drop table flights; diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 7cbc69f2ee7..6f63cc5d0eb 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -1284,6 +1284,7 @@ bool st_select_lex_unit::exec_recursive() sq; sq= sq->next_with_rec_ref) { + sq->reset(); sq->engine->force_reexecution(); } From b132d4d749c29e558b8018f394e57713bab86a7d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 9 Jan 2018 22:52:01 +0000 Subject: [PATCH 17/82] Windows, compilation : Treat warning as error, if MYSQL_MAINTAINER_MODE is set to ERR This matches gcc/clang handling. --- cmake/os/Windows.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index 021f23a14f2..4fd01e5914e 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -142,6 +142,11 @@ IF(MSVC) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996 /we4700 /we4311 /we4477 /we4302 /we4090 /wd4267 ") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099 /we4700 /we4311 /we4477 /we4302 /we4090 /wd4267") + IF(MYSQL_MAINTAINER_MODE MATCHES "ERR") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") + ENDIF() + ENDIF() # Always link with socket library From a408e881cf73d06fc92097fce6ef9584e16edf77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2018 09:17:43 +0200 Subject: [PATCH 18/82] MDEV-14174 crash on start with innodb-track-changed-pages The XtraDB option innodb_track_changed_pages causes the function log_group_read_log_seg() to be invoked even when recv_sys==NULL, leading to the SIGSEGV. This regression was caused by MDEV-11027 InnoDB log recovery is too noisy --- mysql-test/suite/innodb/t/innodb-master.opt | 1 + storage/xtradb/log/log0log.c | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/innodb/t/innodb-master.opt b/mysql-test/suite/innodb/t/innodb-master.opt index 5266978e4f0..8d9b0302a9e 100644 --- a/mysql-test/suite/innodb/t/innodb-master.opt +++ b/mysql-test/suite/innodb/t/innodb-master.opt @@ -2,3 +2,4 @@ --default-storage-engine=MyISAM --innodb-strict-mode=0 --innodb-file-per-table=0 +--loose-innodb-track-changed-pages diff --git a/storage/xtradb/log/log0log.c b/storage/xtradb/log/log0log.c index e327fa7ea9a..bed7e31da28 100644 --- a/storage/xtradb/log/log0log.c +++ b/storage/xtradb/log/log0log.c @@ -2503,7 +2503,6 @@ log_group_read_log_seg( ulint len; ulint source_offset; ibool sync; - ib_time_t time; ut_ad(mutex_own(&(log_sys->mutex))); @@ -2540,13 +2539,16 @@ loop: start_lsn += len; buf += len; - time = ut_time(); + if (recv_sys) { + ib_time_t time = ut_time(); - if (recv_sys->progress_time - time >= 15) { - recv_sys->progress_time = time; - ut_print_timestamp(stderr); - fprintf(stderr, " InnoDB: Read redo log up to LSN=%llu\n", - start_lsn); + if (recv_sys->progress_time - time >= 15) { + recv_sys->progress_time = time; + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Read redo log up to LSN=%llu\n", + start_lsn); + } } if (start_lsn != end_lsn) { From a9c55c0059351220674a1dd3d3bf8d4e884b3285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2018 10:21:52 +0200 Subject: [PATCH 19/82] MDEV-13814 Extra logging when innodb_log_archive=ON Backport the fix from 10.0.33 to 5.5, in case someone compiles XtraDB with -DUNIV_LOG_ARCHIVE --- mysql-test/suite/innodb/t/innodb-master.opt | 1 + storage/xtradb/log/log0log.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/innodb/t/innodb-master.opt b/mysql-test/suite/innodb/t/innodb-master.opt index 8d9b0302a9e..2e71d62206d 100644 --- a/mysql-test/suite/innodb/t/innodb-master.opt +++ b/mysql-test/suite/innodb/t/innodb-master.opt @@ -3,3 +3,4 @@ --innodb-strict-mode=0 --innodb-file-per-table=0 --loose-innodb-track-changed-pages +--loose-innodb-log-archive diff --git a/storage/xtradb/log/log0log.c b/storage/xtradb/log/log0log.c index bed7e31da28..1f9003aa66e 100644 --- a/storage/xtradb/log/log0log.c +++ b/storage/xtradb/log/log0log.c @@ -2539,7 +2539,7 @@ loop: start_lsn += len; buf += len; - if (recv_sys) { + if (recv_recovery_is_on()) { ib_time_t time = ut_time(); if (recv_sys->progress_time - time >= 15) { From d1cf9b167cb0da4987d898cf5d8b389419cae09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2018 13:18:02 +0200 Subject: [PATCH 20/82] MDEV-14909 MariaDB 10.2 refuses to start up after clean shutdown of MariaDB 10.3 recv_log_recover_10_3(): Determine if a log from MariaDB 10.3 is clean. recv_find_max_checkpoint(): Allow startup with a clean 10.3 redo log. srv_prepare_to_delete_redo_log_files(): When starting up with a 10.3 log, display a "Downgrading redo log" message instead of "Upgrading". --- .../r/innodb_encrypt_log_corruption.result | 27 ++++++- .../suite/innodb/r/log_corruption.result | 27 ++++++- mysql-test/suite/innodb/t/log_corruption.test | 62 +++++++++++++- storage/innobase/include/log0log.h | 4 +- storage/innobase/log/log0recv.cc | 80 ++++++++++++++++++- storage/innobase/srv/srv0start.cc | 8 +- 6 files changed, 191 insertions(+), 17 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result index 4a31f1ba454..e4ece7bc4ed 100644 --- a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result +++ b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result @@ -17,6 +17,13 @@ WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and it appears corrupted/ in mysqld.1.err +# empty redo log from before MariaDB 10.2.2 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err # redo log from "after" MariaDB 10.2.2, but with invalid header checksum SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -28,7 +35,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html/ in mysqld.1.err +FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\./ in mysqld.1.err # valid header, but old-format checkpoint blocks SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -86,12 +93,26 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: MLOG_FILE_NAME incorrect:bigot/ in mysqld.1.err FOUND 1 /len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ;/ in mysqld.1.err -# missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +# 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -NOT FOUND /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42$/ in mysqld.1.err +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +# 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +FOUND 1 /Downgrade after a crash is not supported\. The redo log was created with MariaDB 10\.3\.1/ in mysqld.1.err +# Empty 10.3 redo log +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Downgrading redo log:/ in mysqld.1.err # Minimal MariaDB 10.1.21 encrypted redo log SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); diff --git a/mysql-test/suite/innodb/r/log_corruption.result b/mysql-test/suite/innodb/r/log_corruption.result index 3a20a11cd8f..49f6e6b85d2 100644 --- a/mysql-test/suite/innodb/r/log_corruption.result +++ b/mysql-test/suite/innodb/r/log_corruption.result @@ -17,6 +17,13 @@ WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and it appears corrupted/ in mysqld.1.err +# empty redo log from before MariaDB 10.2.2 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err # redo log from "after" MariaDB 10.2.2, but with invalid header checksum SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -28,7 +35,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html/ in mysqld.1.err +FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\./ in mysqld.1.err # valid header, but old-format checkpoint blocks SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -86,12 +93,26 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: MLOG_FILE_NAME incorrect:bigot/ in mysqld.1.err FOUND 1 /len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ;/ in mysqld.1.err -# missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +# 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -NOT FOUND /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42$/ in mysqld.1.err +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +# 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +FOUND 1 /Downgrade after a crash is not supported\. The redo log was created with MariaDB 10\.3\.1/ in mysqld.1.err +# Empty 10.3 redo log +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Downgrading redo log:/ in mysqld.1.err # Minimal MariaDB 10.1.21 encrypted redo log SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test index 8013cc45830..3db15bd43cc 100644 --- a/mysql-test/suite/innodb/t/log_corruption.test +++ b/mysql-test/suite/innodb/t/log_corruption.test @@ -2,7 +2,7 @@ --source include/have_innodb_16k.inc --disable_query_log -call mtr.add_suppression("InnoDB: Upgrade after a crash is not supported"); +call mtr.add_suppression("InnoDB: (Up|Down)grade after a crash is not supported"); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); @@ -16,6 +16,7 @@ call mtr.add_suppression("InnoDB: Log scan aborted at LSN"); call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42\\r?$"); call mtr.add_suppression("InnoDB: Obtaining redo log encryption key version 1 failed"); call mtr.add_suppression("InnoDB: Decrypting checkpoint failed"); +call mtr.add_suppression("InnoDB: Are you sure you are using the right ib_logfiles to start up the database\\? Log sequence number in the ib_logfiles is 1213964,"); --enable_query_log let bugdir= $MYSQLTEST_VARDIR/tmp/log_corruption; @@ -140,6 +141,24 @@ eval $check_no_innodb; let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\\.2\\.2, and it appears corrupted; --source include/search_pattern_in_file.inc +--echo # empty redo log from before MariaDB 10.2.2 +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +die unless seek(OUT, 0x800, 0); +print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 0, 0xb2a); +close OUT or die; +EOF +--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=1m +--source include/start_mysqld.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: Upgrading redo log: +--source include/search_pattern_in_file.inc +--let $restart_parameters= $dirs + --echo # redo log from "after" MariaDB 10.2.2, but with invalid header checksum perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; @@ -165,7 +184,7 @@ EOF --source include/start_mysqld.inc eval $check_no_innodb; --source include/shutdown_mysqld.inc -let SEARCH_PATTERN=InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html; +let SEARCH_PATTERN=InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\.; --source include/search_pattern_in_file.inc --echo # valid header, but old-format checkpoint blocks @@ -321,7 +340,7 @@ let SEARCH_PATTERN=InnoDB: MLOG_FILE_NAME incorrect:bigot; --let SEARCH_PATTERN= len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ; --source include/search_pattern_in_file.inc ---echo # missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +--echo # 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; binmode OUT; @@ -349,7 +368,42 @@ EOF --source include/start_mysqld.inc eval $check_no_innodb; --source include/shutdown_mysqld.inc ---let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42\$ +--let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42 +--source include/search_pattern_in_file.inc + +--echo # 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +print OUT pack("Nx[5]nx[5]", 103, 0x1286), "MariaDB 10.3.1"; +print OUT pack("x[478]N", 0x85021a0f); +close OUT or die; +EOF + +--source include/start_mysqld.inc +eval $check_no_innodb; +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42 +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= Downgrade after a crash is not supported\. The redo log was created with MariaDB 10\.3\.1 +--source include/search_pattern_in_file.inc + +--echo # Empty 10.3 redo log +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +die unless seek(OUT, 0x800, 0); +print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 1, 0x46c8a2a2); +close OUT or die; +EOF + +--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=1m +--source include/start_mysqld.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: Downgrading redo log: --source include/search_pattern_in_file.inc --echo # Minimal MariaDB 10.1.21 encrypted redo log diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 543302c52f0..91ca389df83 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -513,6 +513,8 @@ or the MySQL version that created the redo log file. */ /** The redo log format identifier corresponding to the current format version. Stored in LOG_HEADER_FORMAT. */ #define LOG_HEADER_FORMAT_CURRENT 1 +/** The MariaDB 10.3.2 log format */ +#define LOG_HEADER_FORMAT_10_3 103 /** Encrypted MariaDB redo log */ #define LOG_HEADER_FORMAT_ENCRYPTED (1U<<31) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 6bb2806fb70..5ee6004fe55 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2,7 +2,7 @@ Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -907,6 +907,58 @@ recv_log_format_0_recover(lsn_t lsn) return(DB_SUCCESS); } +/** Determine if a redo log from MariaDB 10.3 is clean. +@return error code +@retval DB_SUCCESS if the redo log is clean +@retval DB_CORRUPTION if the redo log is corrupted +@retval DB_ERROR if the redo log is not empty */ +static +dberr_t +recv_log_recover_10_3() +{ + log_group_t* group = &log_sys->log; + const lsn_t lsn = group->lsn; + const lsn_t source_offset = log_group_calc_lsn_offset(lsn, group); + const ulint page_no + = (ulint) (source_offset / univ_page_size.physical()); + byte* buf = log_sys->buf; + + fil_io(IORequestLogRead, true, + page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no), + univ_page_size, + (ulint) ((source_offset & ~(OS_FILE_LOG_BLOCK_SIZE - 1)) + % univ_page_size.physical()), + OS_FILE_LOG_BLOCK_SIZE, buf, NULL); + + if (log_block_calc_checksum(buf) != log_block_get_checksum(buf)) { + return(DB_CORRUPTION); + } + + if (group->is_encrypted()) { + log_crypt(buf, lsn, OS_FILE_LOG_BLOCK_SIZE, true); + } + + /* On a clean shutdown, the redo log will be logically empty + after the checkpoint lsn. */ + + if (log_block_get_data_len(buf) + != (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) { + return(DB_ERROR); + } + + /* Mark the redo log for downgrading. */ + srv_log_file_size = 0; + recv_sys->parse_start_lsn = recv_sys->recovered_lsn + = recv_sys->scanned_lsn + = recv_sys->mlog_checkpoint_lsn = lsn; + log_sys->last_checkpoint_lsn = log_sys->next_checkpoint_lsn + = log_sys->lsn = log_sys->write_lsn + = log_sys->current_flush_lsn = log_sys->flushed_to_disk_lsn + = lsn; + log_sys->next_checkpoint_no = 0; + return(DB_SUCCESS); +} + /** Find the latest checkpoint in the log header. @param[out] max_field LOG_CHECKPOINT_1 or LOG_CHECKPOINT_2 @return error code or DB_SUCCESS */ @@ -938,18 +990,24 @@ recv_find_max_checkpoint(ulint* max_field) return(DB_CORRUPTION); } + char creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR + 1]; + + memcpy(creator, buf + LOG_HEADER_CREATOR, sizeof creator); + /* Ensure that the string is NUL-terminated. */ + creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR] = 0; + switch (group->format) { case 0: return(recv_find_max_checkpoint_0(&group, max_field)); case LOG_HEADER_FORMAT_CURRENT: case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: + case LOG_HEADER_FORMAT_10_3: + case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED: break; default: - /* Ensure that the string is NUL-terminated. */ - buf[LOG_HEADER_CREATOR_END] = 0; ib::error() << "Unsupported redo log format." " The redo log was created" - " with " << buf + LOG_HEADER_CREATOR << + " with " << creator << ". Please follow the instructions at " REFMAN "upgrading-downgrading.html"; /* Do not issue a message about a possibility @@ -1018,6 +1076,20 @@ recv_find_max_checkpoint(ulint* max_field) return(DB_ERROR); } + switch (group->format) { + case LOG_HEADER_FORMAT_10_3: + case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED: + dberr_t err = recv_log_recover_10_3(); + if (err != DB_SUCCESS) { + ib::error() + << "Downgrade after a crash is not supported." + " The redo log was created with " << creator + << (err == DB_ERROR + ? "." : ", and it appears corrupted."); + } + return(err); + } + return(DB_SUCCESS); } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 886d47e44c0..7bfd1715e9b 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1413,7 +1413,11 @@ srv_prepare_to_delete_redo_log_files( { ib::info info; if (srv_log_file_size == 0) { - info << "Upgrading redo log: "; + info << ((log_sys->log.format + & ~LOG_HEADER_FORMAT_ENCRYPTED) + != LOG_HEADER_FORMAT_10_3 + ? "Upgrading redo log: " + : "Downgrading redo log: "); } else if (n_files != srv_n_log_files || srv_log_file_size != srv_log_file_size_requested) { From dfde5ae912d402ec68fd17eda92112d84a07ffdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2018 13:53:44 +0200 Subject: [PATCH 21/82] MDEV-14130 InnoDB messages should not refer to the MySQL 5.7 manual Replace most occurrences of the REFMAN macro. For some pages there is no replacement yet. --- .../suite/innodb/r/group_commit_crash.result | 2 +- ...oup_commit_crash_no_optimize_thread.result | 2 +- mysql-test/suite/innodb/r/innodb-16k.result | 4 ++-- mysql-test/suite/innodb/r/innodb.result | 12 +++++----- .../suite/innodb/r/innodb_bug47167.result | 8 +++---- .../suite/innodb/r/innodb_file_format.result | 18 +++++++------- .../innodb_prefix_index_restart_server.result | 4 ++-- .../suite/innodb/t/innodb_bug14147491.test | 10 ++++---- mysql-test/suite/innodb_zip/r/16k.result | 4 ++-- mysql-test/suite/innodb_zip/r/4k.result | 4 ++-- mysql-test/suite/innodb_zip/r/8k.result | 4 ++-- .../suite/innodb_zip/r/create_options.result | 20 ++++++++-------- .../innodb_zip/r/index_large_prefix.result | 24 +++++++++---------- .../innodb_zip/r/index_large_prefix_4k.result | 10 ++++---- .../innodb_zip/r/index_large_prefix_8k.result | 20 ++++++++-------- .../suite/innodb_zip/r/innodb-zip.result | 24 +++++++++---------- .../suite/innodb_zip/r/wl6501_scale_1.result | 6 ++--- .../r/innodb_file_format_basic.result | 6 ++--- .../r/innodb_file_format_max_basic.result | 6 ++--- .../r/innodb_large_prefix_basic.result | 10 ++++---- storage/innobase/buf/buf0buf.cc | 6 +---- storage/innobase/gis/gis0sea.cc | 7 +++--- storage/innobase/handler/ha_innodb.cc | 15 ++++++------ storage/innobase/log/log0recv.cc | 4 ++-- storage/innobase/ut/ut0dbg.cc | 4 ++-- 25 files changed, 115 insertions(+), 119 deletions(-) diff --git a/mysql-test/suite/innodb/r/group_commit_crash.result b/mysql-test/suite/innodb/r/group_commit_crash.result index f07df897453..c9daf347ee1 100644 --- a/mysql-test/suite/innodb/r/group_commit_crash.result +++ b/mysql-test/suite/innodb/r/group_commit_crash.result @@ -124,4 +124,4 @@ DROP TABLE t1; DROP TABLE t2; DROP PROCEDURE setcrash; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result b/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result index 2cd9f01d7ed..a3937b5ae3a 100644 --- a/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result +++ b/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result @@ -124,4 +124,4 @@ DROP TABLE t1; DROP TABLE t2; DROP PROCEDURE setcrash; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/innodb-16k.result b/mysql-test/suite/innodb/r/innodb-16k.result index e825e6baa24..a34640310c3 100644 --- a/mysql-test/suite/innodb/r/innodb-16k.result +++ b/mysql-test/suite/innodb/r/innodb-16k.result @@ -1,7 +1,7 @@ call mtr.add_suppression("InnoDB: Cannot add field .* in table .* because after adding it, the row size is .* which is greater than maximum allowed size (.*) for a record on index leaf page."); SET GLOBAL innodb_large_prefix = OFF; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 1) Show the page size from Information Schema SELECT variable_value FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_page_size'; @@ -971,4 +971,4 @@ COL197 TEXT) row_format=compact,ENGINE=INNODB; ERROR 42000: Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/innodb.result b/mysql-test/suite/innodb/r/innodb.result index 6d635fbcdc0..656484af89b 100644 --- a/mysql-test/suite/innodb/r/innodb.result +++ b/mysql-test/suite/innodb/r/innodb.result @@ -2360,13 +2360,13 @@ drop table t1; SET sql_mode = 'NO_ENGINE_SUBSTITUTION'; SET GLOBAL innodb_large_prefix=OFF; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t1 (v varchar(65530), key(v)); Warnings: Warning 1071 Specified key was too long; max key length is 767 bytes SET GLOBAL innodb_large_prefix=default; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ drop table t1; create table t1 (v varchar(65536)); Warnings: @@ -2534,7 +2534,7 @@ drop table t1, t2, t3, t4, t5, t6, t7, t8, t9; SET sql_mode = 'NO_ENGINE_SUBSTITUTION'; SET GLOBAL innodb_large_prefix=OFF; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t1 (col1 varchar(768), index(col1)) character set = latin1 engine = innodb; Warnings: @@ -2553,7 +2553,7 @@ Warnings: Note 1071 Specified key was too long; max key length is 767 bytes SET GLOBAL innodb_large_prefix=default; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -2563,7 +2563,7 @@ t1 CREATE TABLE `t1` ( drop table t1, t2, t3, t4; set global innodb_large_prefix=OFF; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t1 (col1 varchar(768) primary key) character set = latin1 engine = innodb; ERROR 42000: Specified key was too long; max key length is 767 bytes @@ -2579,7 +2579,7 @@ ERROR 42000: Specified key was too long; max key length is 767 bytes SET sql_mode = default; set global innodb_large_prefix=default; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t1 ( id INT PRIMARY KEY diff --git a/mysql-test/suite/innodb/r/innodb_bug47167.result b/mysql-test/suite/innodb/r/innodb_bug47167.result index b678046e308..14977539b3b 100644 --- a/mysql-test/suite/innodb/r/innodb_bug47167.result +++ b/mysql-test/suite/innodb/r/innodb_bug47167.result @@ -4,19 +4,19 @@ select @old_innodb_file_format_max; Barracuda set global innodb_file_format_max = Barracuda; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Barracuda set global innodb_file_format_max = DEFAULT; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Antelope set global innodb_file_format_max = @old_innodb_file_format_max; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Barracuda @@ -29,4 +29,4 @@ ERROR 42000: Variable 'innodb_file_format_max' can't be set to the value of 'ON' set global innodb_file_format_max = off; ERROR 42000: Variable 'innodb_file_format_max' can't be set to the value of 'off' Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/innodb_file_format.result b/mysql-test/suite/innodb/r/innodb_file_format.result index e489911afb5..48f330f90fe 100644 --- a/mysql-test/suite/innodb/r/innodb_file_format.result +++ b/mysql-test/suite/innodb/r/innodb_file_format.result @@ -9,10 +9,10 @@ select @@innodb_file_format_max; Barracuda set global innodb_file_format=antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format=barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format=cheetah; ERROR 42000: Variable 'innodb_file_format' can't be set to the value of 'cheetah' select @@innodb_file_format; @@ -20,7 +20,7 @@ select @@innodb_file_format; Barracuda set global innodb_file_format=default; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format; @@innodb_file_format Barracuda @@ -33,10 +33,10 @@ select @@innodb_file_format; Barracuda set global innodb_file_format_max=antelope; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format_max=barracuda; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format_max=cheetah; ERROR 42000: Variable 'innodb_file_format_max' can't be set to the value of 'cheetah' select @@innodb_file_format_max; @@ -44,7 +44,7 @@ select @@innodb_file_format_max; Barracuda set global innodb_file_format_max=default; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Antelope @@ -57,10 +57,10 @@ select @@innodb_file_format_max; Antelope set global innodb_file_format_max=antelope; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format_check=off; ERROR HY000: Variable 'innodb_file_format_check' is a read only variable Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result b/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result index a3ac78aadce..6fe101c58d7 100644 --- a/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result +++ b/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result @@ -1,6 +1,6 @@ set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE worklog5743 ( col_1_text TEXT(4000) , col_2_text TEXT(4000) , PRIMARY KEY (col_1_text(3072)) @@ -89,4 +89,4 @@ col_1_text = REPEAT("a", 3500) col_2_text = REPEAT("o", 3500) DROP TABLE worklog5743; SET GLOBAL innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index 09f9d53eaff..d8f0cc6d4b6 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -8,12 +8,10 @@ -- source include/not_encrypted.inc --disable_query_log -call mtr.add_suppression("InnoDB: Table `test`.`t1` is corrupted. Please drop the table and recreate."); -call mtr.add_suppression("InnoDB: Cannot open table test/t1 from the internal data dictionary of InnoDB though the .frm file for the table exists. Please refer to http://dev.mysql.com/doc/refman/5.7/en/innodb-troubleshooting.html for how to resolve the issue."); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page \[page id: space=[0-9]+, page number=[0-9]+\]. You may have to recover from a backup."); -call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table.*"); -call mtr.add_suppression("mysqld: Index for table 't1' is corrupt; try to repair it"); -call mtr.add_suppression("mysqld.exe: Index for table 't1' is corrupt; try to repair it"); +call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted\\. Please drop the table and recreate\\."); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page"); +call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table"); +call mtr.add_suppression("Index for table 't1' is corrupt; try to repair it"); --enable_query_log --echo # Create and populate the table to be corrupted diff --git a/mysql-test/suite/innodb_zip/r/16k.result b/mysql-test/suite/innodb_zip/r/16k.result index 0af34c0a738..7010cac182d 100644 --- a/mysql-test/suite/innodb_zip/r/16k.result +++ b/mysql-test/suite/innodb_zip/r/16k.result @@ -284,7 +284,7 @@ Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_file_format = `Antelope`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t4 (id int PRIMARY KEY) ENGINE=innodb KEY_BLOCK_SIZE=8; Got one of the listed errors SHOW WARNINGS; @@ -301,7 +301,7 @@ Error 1005 Can't create table `test`.`t5` (errno: 140 "Wrong create options") Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_format = `Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 7) This series of tests were moved from innodb-index to here # because the second alter table t1 assumes a 16k page size. # Moving the test allows the rest of innodb-index to be run on all diff --git a/mysql-test/suite/innodb_zip/r/4k.result b/mysql-test/suite/innodb_zip/r/4k.result index 5b7a4955a7d..af67087f877 100644 --- a/mysql-test/suite/innodb_zip/r/4k.result +++ b/mysql-test/suite/innodb_zip/r/4k.result @@ -247,7 +247,7 @@ Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_file_format = `Antelope`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t4 (id int PRIMARY KEY) ENGINE=innodb KEY_BLOCK_SIZE=8; ERROR HY000: Can't create table `test`.`t4` (errno: 140 "Wrong create options") SHOW WARNINGS; @@ -266,7 +266,7 @@ Error 1005 Can't create table `test`.`t5` (errno: 140 "Wrong create options") Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_format = `Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 7) Not included here; 16k only # Test 8) Test creating a table that could lead to undo log overflow. CREATE TABLE t1(a blob,b blob,c blob,d blob,e blob,f blob,g blob, diff --git a/mysql-test/suite/innodb_zip/r/8k.result b/mysql-test/suite/innodb_zip/r/8k.result index 2a0245b7859..1b65daeaeac 100644 --- a/mysql-test/suite/innodb_zip/r/8k.result +++ b/mysql-test/suite/innodb_zip/r/8k.result @@ -259,7 +259,7 @@ Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_file_format = `Antelope`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t4 (id int PRIMARY KEY) ENGINE=innodb KEY_BLOCK_SIZE=8; ERROR HY000: Can't create table `test`.`t4` (errno: 140 "Wrong create options") SHOW WARNINGS; @@ -277,7 +277,7 @@ Error 1005 Can't create table `test`.`t5` (errno: 140 "Wrong create options") Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_format = `Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 7) Not included here; 16k only # Test 8) Test creating a table that could lead to undo log overflow. CREATE TABLE t1(a blob,b blob,c blob,d blob,e blob,f blob,g blob, diff --git a/mysql-test/suite/innodb_zip/r/create_options.result b/mysql-test/suite/innodb_zip/r/create_options.result index 4e38bec08e3..9165e9ee5ae 100644 --- a/mysql-test/suite/innodb_zip/r/create_options.result +++ b/mysql-test/suite/innodb_zip/r/create_options.result @@ -1,7 +1,7 @@ SET default_storage_engine=InnoDB; SET GLOBAL innodb_file_format=`Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET GLOBAL innodb_file_per_table=ON; SET SESSION innodb_strict_mode = ON; # Test 1) StrictMode=ON, CREATE and ALTER with each ROW_FORMAT & KEY_BLOCK_SIZE=0 @@ -264,7 +264,7 @@ Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB # and that they can be set to default values during strict mode. SET GLOBAL innodb_file_format=Antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t1 ( i INT ) KEY_BLOCK_SIZE=4; Got one of the listed errors SHOW WARNINGS; @@ -318,12 +318,12 @@ Warning 1478 InnoDB: ROW_FORMAT=DYNAMIC requires innodb_file_format > Antelope. Error 1478 Table storage engine 'InnoDB' does not support the create option 'ROW_FORMAT' SET GLOBAL innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ DROP TABLE t1; CREATE TABLE t1 ( i INT ) ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4; SET GLOBAL innodb_file_format=Antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f1 INT; ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ROW_FORMAT' SHOW WARNINGS; @@ -344,7 +344,7 @@ SHOW WARNINGS; Level Code Message SET GLOBAL innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 8) StrictMode=ON, Make sure ROW_FORMAT=COMPRESSED # and a valid non-zero KEY_BLOCK_SIZE are rejected with # innodb_file_per_table=OFF and that they can be set to default @@ -754,7 +754,7 @@ TABLE_NAME ROW_FORMAT CREATE_OPTIONS t1 Compressed row_format=COMPRESSED key_block_size=1 SET GLOBAL innodb_file_format=Antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f1 INT; Warnings: Warning 1478 InnoDB: KEY_BLOCK_SIZE requires innodb_file_format > Antelope. @@ -772,7 +772,7 @@ TABLE_NAME ROW_FORMAT CREATE_OPTIONS t1 Dynamic row_format=COMPRESSED key_block_size=1 SET GLOBAL innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f2 INT; SHOW WARNINGS; Level Code Message @@ -788,7 +788,7 @@ TABLE_NAME ROW_FORMAT CREATE_OPTIONS t1 Dynamic row_format=DYNAMIC SET GLOBAL innodb_file_format=Antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f1 INT; SHOW WARNINGS; Level Code Message @@ -797,7 +797,7 @@ TABLE_NAME ROW_FORMAT CREATE_OPTIONS t1 Dynamic row_format=DYNAMIC SET GLOBAL innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f2 INT; SHOW WARNINGS; Level Code Message @@ -852,4 +852,4 @@ t1 Dynamic row_format=DYNAMIC # Cleanup DROP TABLE t1; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb_zip/r/index_large_prefix.result b/mysql-test/suite/innodb_zip/r/index_large_prefix.result index df177e8ea2a..fe03586546a 100644 --- a/mysql-test/suite/innodb_zip/r/index_large_prefix.result +++ b/mysql-test/suite/innodb_zip/r/index_large_prefix.result @@ -96,7 +96,7 @@ create table worklog5743_8(a1 int, a2 TEXT, a3 TEXT) KEY_BLOCK_SIZE=8; create table worklog5743_16(a1 int, a2 TEXT, a3 TEXT) KEY_BLOCK_SIZE=16; set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_1(a2(4000)); Got one of the listed errors show warnings; @@ -105,7 +105,7 @@ Note 1071 Specified key was too long; max key length is 767 bytes Error 1118 Row size too large. The maximum row size for the used table type, not counting BLOBs, is 8126. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_1(a2(4000)); Got one of the listed errors show warnings; @@ -130,7 +130,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET sql_mode= ''; create index idx1 on worklog5743_2(a2(4000)); Warnings: @@ -140,7 +140,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_2(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 8126. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -165,7 +165,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_4(a2(4000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -174,7 +174,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_4(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 8126. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -199,7 +199,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_8(a2(1000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -208,7 +208,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_8(a2(3073)); Warnings: Note 1071 Specified key was too long; max key length is 3072 bytes @@ -239,7 +239,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_16(a2(1000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -248,7 +248,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_16(a2(3073)); Warnings: Note 1071 Specified key was too long; max key length is 3072 bytes @@ -285,7 +285,7 @@ insert into worklog5743_8 values(9, repeat("a", 10000), repeat("a", 10000)); insert into worklog5743_16 values(9, repeat("a", 10000), repeat("a", 10000)); set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ insert into worklog5743_1 values(2, repeat("b", 10000)); insert into worklog5743_2 values(2, repeat("b", 10000)); insert into worklog5743_4 values(2, repeat("b", 10000)); @@ -293,7 +293,7 @@ insert into worklog5743_8 values(2, repeat("b", 10000), repeat("b", 10000)); insert into worklog5743_16 values(2, repeat("b", 10000), repeat("b", 10000)); set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select a1, left(a2, 20) from worklog5743_1; a1 left(a2, 20) 9 aaaaaaaaaaaaaaaaaaaa diff --git a/mysql-test/suite/innodb_zip/r/index_large_prefix_4k.result b/mysql-test/suite/innodb_zip/r/index_large_prefix_4k.result index 50d4854154b..c2929bf7937 100644 --- a/mysql-test/suite/innodb_zip/r/index_large_prefix_4k.result +++ b/mysql-test/suite/innodb_zip/r/index_large_prefix_4k.result @@ -99,7 +99,7 @@ create table worklog5743_2(a1 int, a2 TEXT not null) KEY_BLOCK_SIZE=2; create table worklog5743_4(a1 int, a2 TEXT not null) KEY_BLOCK_SIZE=4; set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_1(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 1982. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -124,7 +124,7 @@ show warnings; Level Code Message set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET sql_mode= ''; create index idx1 on worklog5743_2(a2(4000)); Warnings: @@ -161,7 +161,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_4(a2(4000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -201,13 +201,13 @@ insert into worklog5743_2 values(9, repeat("a", 10000)); insert into worklog5743_4 values(9, repeat("a", 10000)); set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ insert into worklog5743_1 values(2, repeat("b", 10000)); insert into worklog5743_2 values(2, repeat("b", 10000)); insert into worklog5743_4 values(2, repeat("b", 10000)); set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select a1, left(a2, 20) from worklog5743_1; a1 left(a2, 20) 9 aaaaaaaaaaaaaaaaaaaa diff --git a/mysql-test/suite/innodb_zip/r/index_large_prefix_8k.result b/mysql-test/suite/innodb_zip/r/index_large_prefix_8k.result index e99bb0876a5..60dafa2ac46 100644 --- a/mysql-test/suite/innodb_zip/r/index_large_prefix_8k.result +++ b/mysql-test/suite/innodb_zip/r/index_large_prefix_8k.result @@ -100,7 +100,7 @@ create table worklog5743_4(a1 int, a2 TEXT not null) KEY_BLOCK_SIZE=4; create table worklog5743_8(a1 int, a2 TEXT, a3 TEXT) KEY_BLOCK_SIZE=8; set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_1(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 4030. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -109,7 +109,7 @@ Note 1071 Specified key was too long; max key length is 767 bytes Error 1118 Row size too large. The maximum row size for the used table type, not counting BLOBs, is 4030. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_1(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 4030. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -134,7 +134,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET sql_mode= ''; create index idx1 on worklog5743_2(a2(4000)); Warnings: @@ -144,7 +144,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_2(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 4030. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -169,7 +169,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_4(a2(4000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -178,7 +178,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx3 on worklog5743_4(a2(1537)); Warnings: Note 1071 Specified key was too long; max key length is 1536 bytes @@ -201,7 +201,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_8(a2(1000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -210,7 +210,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_8(a2(3073)); Warnings: Note 1071 Specified key was too long; max key length is 1536 bytes @@ -238,14 +238,14 @@ insert into worklog5743_4 values(9, repeat("a", 10000)); insert into worklog5743_8 values(9, repeat("a", 10000), repeat("a", 10000)); set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ insert into worklog5743_1 values(2, repeat("b", 10000)); insert into worklog5743_2 values(2, repeat("b", 10000)); insert into worklog5743_4 values(2, repeat("b", 10000)); insert into worklog5743_8 values(2, repeat("b", 10000), repeat("b", 10000)); set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select a1, left(a2, 20) from worklog5743_1; a1 left(a2, 20) 9 aaaaaaaaaaaaaaaaaaaa diff --git a/mysql-test/suite/innodb_zip/r/innodb-zip.result b/mysql-test/suite/innodb_zip/r/innodb-zip.result index c715f77b9ba..419671a6735 100644 --- a/mysql-test/suite/innodb_zip/r/innodb-zip.result +++ b/mysql-test/suite/innodb_zip/r/innodb-zip.result @@ -9,7 +9,7 @@ set session innodb_strict_mode=0; set global innodb_file_per_table=off; set global innodb_file_format=`0`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET @@global.innodb_stats_on_metadata=ON; create table t0(a int primary key) engine=innodb row_format=compressed; Warnings: @@ -45,7 +45,7 @@ Warning 1478 InnoDB: KEY_BLOCK_SIZE requires innodb_file_format > Antelope. Warning 1478 InnoDB: ignoring KEY_BLOCK_SIZE=1. set global innodb_file_format=`1`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t7(a int primary key) engine=innodb key_block_size=1 row_format=redundant; Warnings: @@ -161,13 +161,13 @@ update t1 set c3 = repeat('E', 20000) where c1 = 1; drop table t1; set global innodb_file_format=`0`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format; @@innodb_file_format Antelope set global innodb_file_format=`1`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format; @@innodb_file_format Barracuda @@ -177,10 +177,10 @@ set global innodb_file_format=`-1`; ERROR 42000: Variable 'innodb_file_format' can't be set to the value of '-1' set global innodb_file_format=`Antelope`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format=`Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format=`Cheetah`; ERROR 42000: Variable 'innodb_file_format' can't be set to the value of 'Cheetah' set global innodb_file_format=`abc`; @@ -192,7 +192,7 @@ ERROR 42000: Variable 'innodb_file_format' can't be set to the value of '' set global innodb_file_per_table = on; set global innodb_file_format = `1`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set innodb_strict_mode = off; create table t1 (id int primary key) engine = innodb key_block_size = 0; drop table t1; @@ -328,7 +328,7 @@ drop table t7, t8, t9; set global innodb_file_per_table = on; set global innodb_file_format = `0`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t1 (id int primary key) engine = innodb key_block_size = 1; ERROR HY000: Can't create table `mysqltest_innodb_zip`.`t1` (errno: 140 "Wrong create options") show warnings; @@ -374,14 +374,14 @@ drop table t8, t9; set global innodb_file_per_table=1; set global innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_per_table=on; set global innodb_file_format=`Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format_max=`Antelope`; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table normal_table ( c1 int ) engine = innodb; @@ -396,7 +396,7 @@ select @@innodb_file_format_max; Barracuda set global innodb_file_format_max=`Antelope`; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Antelope diff --git a/mysql-test/suite/innodb_zip/r/wl6501_scale_1.result b/mysql-test/suite/innodb_zip/r/wl6501_scale_1.result index 0064cf2a469..3a74b6ebc11 100644 --- a/mysql-test/suite/innodb_zip/r/wl6501_scale_1.result +++ b/mysql-test/suite/innodb_zip/r/wl6501_scale_1.result @@ -111,7 +111,7 @@ drop procedure populate; drop procedure populate_small; set global innodb_file_format = Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_per_table = 1; set innodb_strict_mode=OFF; create procedure populate() @@ -224,7 +224,7 @@ drop procedure populate; drop procedure populate_small; set global innodb_file_format = Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_per_table = 1; set innodb_strict_mode=OFF; create procedure populate() @@ -341,5 +341,5 @@ drop procedure populate; drop procedure populate_small; set global innodb_file_format = Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_per_table = 1; diff --git a/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result b/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result index c330bbf5c16..0b2cbb97d25 100644 --- a/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result @@ -25,7 +25,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_FILE_FORMAT Barracuda set global innodb_file_format='Antelope'; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_file_format; @@global.innodb_file_format Antelope @@ -37,7 +37,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_FILE_FORMAT Antelope set @@global.innodb_file_format='Barracuda'; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_file_format; @@global.innodb_file_format Barracuda @@ -59,7 +59,7 @@ set global innodb_file_format='Salmon'; ERROR 42000: Variable 'innodb_file_format' can't be set to the value of 'Salmon' SET @@global.innodb_file_format = @start_global_value; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_file_format; @@global.innodb_file_format Barracuda diff --git a/mysql-test/suite/sys_vars/r/innodb_file_format_max_basic.result b/mysql-test/suite/sys_vars/r/innodb_file_format_max_basic.result index 5402e16a424..d36a17933d4 100644 --- a/mysql-test/suite/sys_vars/r/innodb_file_format_max_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_file_format_max_basic.result @@ -27,7 +27,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_FILE_FORMAT_MAX Barracuda SET global innodb_file_format_max='Antelope'; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_file_format_max; @@global.innodb_file_format_max Antelope @@ -41,7 +41,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_FILE_FORMAT_MAX Antelope SET @@global.innodb_file_format_max='Barracuda'; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_file_format_max; @@global.innodb_file_format_max Barracuda @@ -65,7 +65,7 @@ SET global innodb_file_format_max='Salmon'; ERROR 42000: Variable 'innodb_file_format_max' can't be set to the value of 'Salmon' SET @@global.innodb_file_format_max = @start_global_value; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_file_format_max; @@global.innodb_file_format_max Barracuda diff --git a/mysql-test/suite/sys_vars/r/innodb_large_prefix_basic.result b/mysql-test/suite/sys_vars/r/innodb_large_prefix_basic.result index c6e803ffef8..17a23081096 100644 --- a/mysql-test/suite/sys_vars/r/innodb_large_prefix_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_large_prefix_basic.result @@ -25,7 +25,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_LARGE_PREFIX ON set global innodb_large_prefix='OFF'; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_large_prefix; @@global.innodb_large_prefix 0 @@ -37,7 +37,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_LARGE_PREFIX OFF set @@global.innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_large_prefix; @@global.innodb_large_prefix 1 @@ -49,7 +49,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_LARGE_PREFIX ON set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_large_prefix; @@global.innodb_large_prefix 0 @@ -61,7 +61,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_LARGE_PREFIX OFF set @@global.innodb_large_prefix='ON'; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_large_prefix; @@global.innodb_large_prefix 1 @@ -96,7 +96,7 @@ set global innodb_large_prefix='AUTO'; ERROR 42000: Variable 'innodb_large_prefix' can't be set to the value of 'AUTO' SET @@global.innodb_large_prefix = @start_global_value; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_large_prefix; @@global.innodb_large_prefix 1 diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ae081fa39da..07d3f7efe27 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4360,11 +4360,7 @@ loop: << ". The most probable cause" " of this error may be that the" " table has been corrupted." - " You can try to fix this" - " problem by using" - " innodb_force_recovery." - " Please see " REFMAN " for more" - " details. Aborting..."; + " See https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/"; } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 87ebd9ad34a..dcf8cc6f781 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -773,8 +773,9 @@ rtr_page_get_father_node_ptr( error << ". You should dump + drop + reimport the table to" " fix the corruption. If the crash happens at" - " database startup, see " REFMAN - "forcing-innodb-recovery.html about forcing" + " database startup, see " + "https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/" + " about forcing" " recovery. Then dump + drop + reimport."; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4a0b6dfe4b9..b42da1c025e 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. @@ -3654,7 +3654,7 @@ static uint innobase_partition_flags() #define DEPRECATED_FORMAT_PARAMETER(x) \ "Using " x " is deprecated and the parameter" \ " may be removed in future releases." \ - " See " REFMAN "innodb-file-format.html" + " See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/" /** Deprecation message about innodb_file_format */ static const char* deprecated_file_format @@ -22573,7 +22573,8 @@ const char* BUG_REPORT_MSG = "Submit a detailed bug report to https://jira.mariadb.org/"; const char* FORCE_RECOVERY_MSG = - "Please refer to " REFMAN "forcing-innodb-recovery.html" + "Please refer to " + "https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/" " for information about forcing recovery."; const char* ERROR_CREATING_MSG = @@ -22581,17 +22582,17 @@ const char* ERROR_CREATING_MSG = const char* OPERATING_SYSTEM_ERROR_MSG = "Some operating system error numbers are described at" - " " REFMAN "operating-system-error-codes.html"; + " https://mariadb.com/kb/en/library/operating-system-error-codes/"; const char* FOREIGN_KEY_CONSTRAINTS_MSG = - "Please refer to " REFMAN "innodb-foreign-key-constraints.html" + "Please refer to https://mariadb.com/kb/en/library/foreign-keys/" " for correct foreign key definition."; const char* SET_TRANSACTION_MSG = - "Please refer to " REFMAN "set-transaction.html"; + "Please refer to https://mariadb.com/kb/en/library/set-transaction/"; const char* INNODB_PARAMETERS_MSG = - "Please refer to " REFMAN "innodb-parameters.html"; + "Please refer to https://mariadb.com/kb/en/library/xtradbinnodb-server-system-variables/"; /********************************************************************** Converts an identifier from my_charset_filename to UTF-8 charset. diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 5ee6004fe55..818a0f0422e 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -842,7 +842,7 @@ recv_find_max_checkpoint_0(log_group_t** max_group, ulint* max_field) " This redo log was created before MariaDB 10.2.2," " and we did not find a valid checkpoint." " Please follow the instructions at" - " " REFMAN "upgrading.html"; + " https://mariadb.com/kb/en/library/upgrading/"; return(DB_ERROR); } @@ -869,7 +869,7 @@ recv_log_format_0_recover(lsn_t lsn) " This redo log was created before MariaDB 10.2.2"; static const char* NO_UPGRADE_RTFM_MSG = ". Please follow the instructions at " - REFMAN "upgrading.html"; + "https://mariadb.com/kb/en/library/upgrading/"; fil_io(IORequestLogRead, true, page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no), diff --git a/storage/innobase/ut/ut0dbg.cc b/storage/innobase/ut/ut0dbg.cc index 9e596dcda81..7df189ac560 100644 --- a/storage/innobase/ut/ut0dbg.cc +++ b/storage/innobase/ut/ut0dbg.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,7 +53,7 @@ ut_dbg_assertion_failed( " or crashes, even\n" "InnoDB: immediately after the mysqld startup, there may be\n" "InnoDB: corruption in the InnoDB tablespace. Please refer to\n" - "InnoDB: " REFMAN "forcing-innodb-recovery.html\n" + "InnoDB: https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/\n" "InnoDB: about forcing recovery.\n", stderr); fflush(stderr); From 1a62c8a39647db00283b4e35fb6db3c7bc31e3ca Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 10 Jan 2018 14:41:10 +0200 Subject: [PATCH 22/82] MDEV-14822 binlog.binlog_killed fails with wrong result Problem was timing between the thread that was killed and reading the binary log. Updated the test to wait until the killed thread was properly terminated before checking what's in the binary log. To make check safe, I changed "threads_connected" to be updated after thd::cleanup() is done, to ensure that all binary logs updates are done before the variable is changed. This was mainly done to get the test deterministic and have now other real influence in how the server works. --- .../suite/binlog/r/binlog_killed.result | 42 +++++++++---------- mysql-test/suite/binlog/t/binlog_killed.test | 34 ++++++++++----- sql/mysqld.cc | 7 ++-- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/mysql-test/suite/binlog/r/binlog_killed.result b/mysql-test/suite/binlog/r/binlog_killed.result index d751b1f9002..ae851c38e5f 100644 --- a/mysql-test/suite/binlog/r/binlog_killed.result +++ b/mysql-test/suite/binlog/r/binlog_killed.result @@ -185,90 +185,88 @@ connection con3; MI: MyISAM, INNODB BEGIN; INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 2); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Format_desc # # SERVER_VERSION, BINLOG_VERSION master-bin.000002 # Gtid_list # # [#-#-#] -master-bin.000002 # Binlog_checkpoint # # master-bin.000001 -master-bin.000002 # Binlog_checkpoint # # master-bin.000002 master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=3 master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=4 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 2) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; IM: INNODB, MyISAM BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 3); +INSERT INTO t2 VALUES (NULL, 4); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=4 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 4) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=5 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 3) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; IMI: INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 5); +INSERT INTO t2 VALUES (NULL, 6); +INSERT INTO t1 VALUES (NULL, 7); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=5 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 6) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=6 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 5) master-bin.000002 # Intvar # # INSERT_ID=7 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 7) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; MI2: MyISAM, INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t2 VALUES (NULL, 8); +INSERT INTO t1 VALUES (NULL, 9); +INSERT INTO t2 VALUES (NULL, 10); +INSERT INTO t1 VALUES (NULL, 11); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=6 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 8) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=7 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 10) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=8 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 9) master-bin.000002 # Intvar # # INSERT_ID=9 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 11) master-bin.000002 # Query # # ROLLBACK connection default; disconnect con1; diff --git a/mysql-test/suite/binlog/t/binlog_killed.test b/mysql-test/suite/binlog/t/binlog_killed.test index 8ac223603a3..7c3a262d2c1 100644 --- a/mysql-test/suite/binlog/t/binlog_killed.test +++ b/mysql-test/suite/binlog/t/binlog_killed.test @@ -10,6 +10,13 @@ call mtr.add_suppression("Unsafe statement written to the binary log using state -- source include/not_embedded.inc +# +# Avoid printing binlog checkpoints +# + +--let $skip_checkpoint_events=1 + + ### ### bug#22725 : incorrect killed error in binlogged query ### @@ -362,16 +369,20 @@ connect (con3, localhost, root,,); connection con3; let $ID= `select connection_id()`; +--let $threads_connected=`select variable_value from information_schema.global_status where variable_name="threads_connected"` +--let wait_condition=select variable_value < $threads_connected from information_schema.global_status where variable_name="threads_connected" + --echo MI: MyISAM, INNODB BEGIN; INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 2); #Connection con1 as killer throughout the test connection con1; --replace_result $ID ID --eval KILL $ID let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1); +--source include/wait_condition.inc --let $binlog_start= 4 --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -383,12 +394,13 @@ let $ID= `select connection_id()`; --echo IM: INNODB, MyISAM BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 3); +INSERT INTO t2 VALUES (NULL, 4); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -400,13 +412,14 @@ let $ID= `select connection_id()`; --echo IMI: INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 5); +INSERT INTO t2 VALUES (NULL, 6); +INSERT INTO t1 VALUES (NULL, 7); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -418,14 +431,15 @@ let $ID= `select connection_id()`; --echo MI2: MyISAM, INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t2 VALUES (NULL, 8); +INSERT INTO t1 VALUES (NULL, 9); +INSERT INTO t2 VALUES (NULL, 10); +INSERT INTO t1 VALUES (NULL, 11); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0ec99d6eb96..c2f0bbf2f10 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2919,16 +2919,17 @@ void unlink_thd(THD *thd) DBUG_ENTER("unlink_thd"); DBUG_PRINT("enter", ("thd: %p", thd)); + thd->cleanup(); + thd->add_status_to_global(); + unlink_not_visible_thd(thd); + /* Do not decrement when its wsrep system thread. wsrep_applier is set for applier as well as rollbacker threads. */ if (IF_WSREP(!thd->wsrep_applier, 1)) dec_connection_count(thd->scheduler); - thd->cleanup(); - thd->add_status_to_global(); - unlink_not_visible_thd(thd); thd->free_connection(); DBUG_VOID_RETURN; From 9c9cf556a15af84f9133ef859b50596085f7ae8e Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 6 Oct 2017 17:52:35 +0200 Subject: [PATCH 23/82] MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in Roll back to most general duplicate removing strategi in case of different stratagies for one position. --- mysql-test/r/subselect.result | 26 +++++++++ mysql-test/r/subselect_no_mat.result | 26 +++++++++ mysql-test/r/subselect_no_opts.result | 26 +++++++++ mysql-test/r/subselect_no_scache.result | 26 +++++++++ mysql-test/r/subselect_no_semijoin.result | 26 +++++++++ mysql-test/t/subselect.test | 28 ++++++++++ sql/opt_subselect.cc | 64 +++++++++++++++++------ 7 files changed, 206 insertions(+), 16 deletions(-) diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index 8fc9cd38728..e2f2b6521c8 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -7146,3 +7146,29 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result index a71df97e6bc..25ef4a76962 100644 --- a/mysql-test/r/subselect_no_mat.result +++ b/mysql-test/r/subselect_no_mat.result @@ -7143,6 +7143,32 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests set optimizer_switch=default; select @@optimizer_switch like '%materialization=on%'; @@optimizer_switch like '%materialization=on%' diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result index 2f8c67fa167..074874fbd5b 100644 --- a/mysql-test/r/subselect_no_opts.result +++ b/mysql-test/r/subselect_no_opts.result @@ -7141,4 +7141,30 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests set @optimizer_switch_for_subselect_test=null; diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result index 8dda0a4fd36..de49585b562 100644 --- a/mysql-test/r/subselect_no_scache.result +++ b/mysql-test/r/subselect_no_scache.result @@ -7152,6 +7152,32 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests set optimizer_switch=default; select @@optimizer_switch like '%subquery_cache=on%'; @@optimizer_switch like '%subquery_cache=on%' diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result index 339e2b89786..46a46c91ddc 100644 --- a/mysql-test/r/subselect_no_semijoin.result +++ b/mysql-test/r/subselect_no_semijoin.result @@ -7141,5 +7141,31 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests set @optimizer_switch_for_subselect_test=null; set @join_cache_level_for_subselect_test=NULL; diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index 50d9517043a..e6233e9de78 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -6033,3 +6033,31 @@ CREATE TABLE t2 (f2 INT, KEY(f2)) ENGINE=MyISAM; INSERT t2 VALUES (6),(9); SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; drop table t1, t2; + +--echo # +--echo # MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +--echo # (5.5 test) +--echo # +SET @optimiser_switch_save= @@optimizer_switch; + +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); + +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); + +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); + +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); + +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); + +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; + +--echo End of 5.5 tests diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index 69a5367cdf1..028bf44bf79 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -2687,7 +2687,8 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, LooseScan detector in best_access_path) */ remaining_tables &= ~new_join_tab->table->map; - table_map dups_producing_tables; + table_map dups_producing_tables, prev_dups_producing_tables, + prev_sjm_lookup_tables; if (idx == join->const_tables) dups_producing_tables= 0; @@ -2698,7 +2699,7 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, if ((emb_sj_nest= new_join_tab->emb_sj_nest)) dups_producing_tables |= emb_sj_nest->sj_inner_tables; - Semi_join_strategy_picker **strategy; + Semi_join_strategy_picker **strategy, **prev_strategy; if (idx == join->const_tables) { /* First table, initialize pickers */ @@ -2750,23 +2751,54 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, 3. We have no clue what to do about fanount of semi-join Y. */ if ((dups_producing_tables & handled_fanout) || - (read_time < *current_read_time && + (read_time < *current_read_time && !(handled_fanout & pos->inner_tables_handled_with_other_sjs))) { - /* Mark strategy as used */ - (*strategy)->mark_used(); - pos->sj_strategy= sj_strategy; - if (sj_strategy == SJ_OPT_MATERIALIZE) - join->sjm_lookup_tables |= handled_fanout; + DBUG_ASSERT(pos->sj_strategy != sj_strategy); + /* + If the strategy choosen first time or + the strategy replace strategy which was used to exectly the same + tables + */ + if (pos->sj_strategy == SJ_OPT_NONE || + handled_fanout == + (prev_dups_producing_tables ^ dups_producing_tables)) + { + prev_strategy= strategy; + if (pos->sj_strategy == SJ_OPT_NONE) + { + prev_dups_producing_tables= dups_producing_tables; + prev_sjm_lookup_tables= join->sjm_lookup_tables; + } + /* Mark strategy as used */ + (*strategy)->mark_used(); + pos->sj_strategy= sj_strategy; + if (sj_strategy == SJ_OPT_MATERIALIZE) + join->sjm_lookup_tables |= handled_fanout; + else + join->sjm_lookup_tables &= ~handled_fanout; + *current_read_time= read_time; + *current_record_count= rec_count; + dups_producing_tables &= ~handled_fanout; + //TODO: update bitmap of semi-joins that were handled together with + // others. + if (is_multiple_semi_joins(join, join->positions, idx, + handled_fanout)) + pos->inner_tables_handled_with_other_sjs |= handled_fanout; + } else - join->sjm_lookup_tables &= ~handled_fanout; - *current_read_time= read_time; - *current_record_count= rec_count; - dups_producing_tables &= ~handled_fanout; - //TODO: update bitmap of semi-joins that were handled together with - // others. - if (is_multiple_semi_joins(join, join->positions, idx, handled_fanout)) - pos->inner_tables_handled_with_other_sjs |= handled_fanout; + { + /* Conflict fall to most general variant */ + (*prev_strategy)->set_empty(); + dups_producing_tables= prev_dups_producing_tables; + join->sjm_lookup_tables= prev_sjm_lookup_tables; + // mark it 'none' to avpoid loops + pos->sj_strategy= SJ_OPT_NONE; + // next skip to last; + strategy= pickers + + (sizeof(pickers)/sizeof(Semi_join_strategy_picker*) - 3); + continue; + } } else { From ec97aba28462aab18137f96317c4b441385df12a Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 10 Jan 2018 19:36:38 +0200 Subject: [PATCH 24/82] Fixed BUILD scripts - Skip 'clean' if not a git repository (Good for tar files) - Add configuration for ASAN builds --- BUILD/FINISH.sh | 12 ++++++---- BUILD/compile-pentium64-asan-max | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) create mode 100755 BUILD/compile-pentium64-asan-max diff --git a/BUILD/FINISH.sh b/BUILD/FINISH.sh index 20784c49f0b..1faf62a9b62 100644 --- a/BUILD/FINISH.sh +++ b/BUILD/FINISH.sh @@ -14,7 +14,7 @@ # License along with this library; if not, write to the Free # Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, # MA 02110-1301, USA - +set -x -v cflags="$c_warnings $extra_flags $EXTRA_FLAGS $EXTRA_CFLAGS" cxxflags="$cxx_warnings $base_cxxflags $extra_flags $EXTRA_FLAGS $EXTRA_CXXFLAGS" extra_configs="$extra_configs $local_infile_configs $EXTRA_CONFIGS" @@ -32,15 +32,19 @@ then configure="$configure --verbose" fi +commands="" # git clean -fdX removes all ignored (build) files -commands="\ +if test -d .git +then + commands="\ git clean -fdX cd ./libmariadb git submodule update cd ../storage/rocksdb/rocksdb git submodule update -cd ../../.. - +cd ../../.." +fi +commands="$commands path=`dirname $0` . \"$path/autorun.sh\"" diff --git a/BUILD/compile-pentium64-asan-max b/BUILD/compile-pentium64-asan-max new file mode 100755 index 00000000000..db595b418df --- /dev/null +++ b/BUILD/compile-pentium64-asan-max @@ -0,0 +1,39 @@ +#! /bin/sh +# Copyright (c) 2005, 2010, Oracle and/or its affiliates. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +# Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +path=`dirname $0` +. "$path/SETUP.sh" + +extra_flags="$pentium64_cflags $debug_cflags -lasan -O -g -fsanitize=address" +extra_configs="$pentium_configs $debug_configs $valgrind_configs $max_configs" +export LDFLAGS="-ldl" + +. "$path/FINISH.sh" From 79fc07471068dfe41d08cd23dff100039949836f Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 10 Jan 2018 16:43:25 +0000 Subject: [PATCH 25/82] Update CONC --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index bc1777f8ce8..8231a0add6a 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit bc1777f8ce8cb5fc3b4f0e184d56871d5b809c20 +Subproject commit 8231a0add6a9c2e12a758db2e7072d42deef354b From cdb7a8fa6928f3fb103ed7f66486dc9130a60e89 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 10 Jan 2018 20:16:52 +0000 Subject: [PATCH 26/82] Silence warning coming from Windows' own header dbghelp.h --- mysql-test/lib/My/SafeProcess/safe_kill_win.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc index bb884cba11e..6ca38ceee81 100644 --- a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc +++ b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc @@ -26,7 +26,20 @@ #include #include #include + +#ifdef _MSC_VER +/* Silence warning in OS header dbghelp.h */ +#pragma warning(push) +#pragma warning(disable : 4091) +#endif + #include + +#ifdef _MSC_VER +/* Silence warning in OS header dbghelp.h */ +#pragma warning(pop) +#endif + #include #include From bdcd7f79e4a33e08bae31a86cee98e23ff6824ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 09:33:26 +0200 Subject: [PATCH 27/82] MDEV-14916 InnoDB reports warning for "Purge reached the head of the history list" The warning was originally added in commit c67663054a7db1c77dd1dbc85b63595667a53500 (MySQL 4.1.12, 5.0.3) to trace claimed undo log corruption that was analyzed in https://lists.mysql.com/mysql/176250 on November 9, 2004. Originally, the limit was 20,000 undo log headers or transactions, but in commit 9d6d1902e091c868bb288e0ccf9f975ccb474db9 in MySQL 5.5.11 it was increased to 2,000,000. The message can be triggered when the progress of purge is prevented by a long-running transaction (or just an idle transaction whose read view was started a long time ago), by running many transactions that UPDATE or DELETE some records, then starting another transaction with a read view, and finally by executing more than 2,000,000 transactions that UPDATE or DELETE records in InnoDB tables. Finally, when the oldest long-running transaction is completed, purge would run up to the next-oldest transaction, and there would still be more than 2,000,000 transactions to purge. Because the message can be triggered when the database is obviously not corrupted, it should be removed. Heavy users of InnoDB should be monitoring the "History list length" in SHOW ENGINE INNODB STATUS; there is no need to spam the error log. --- storage/innobase/trx/trx0purge.c | 26 -------------------------- storage/xtradb/trx/trx0purge.c | 26 -------------------------- 2 files changed, 52 deletions(-) diff --git a/storage/innobase/trx/trx0purge.c b/storage/innobase/trx/trx0purge.c index 59f8cd52afa..d23d1fc6a1f 100644 --- a/storage/innobase/trx/trx0purge.c +++ b/storage/innobase/trx/trx0purge.c @@ -720,32 +720,6 @@ trx_purge_rseg_get_next_history_log( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - - mutex_enter(&kernel_mutex); - - /* Add debug code to track history list corruption reported - on the MySQL mailing list on Nov 9, 2004. The fut0lst.c - file-based list was corrupt. The prev node pointer was - FIL_NULL, even though the list length was over 8 million nodes! - We assume that purge truncates the history list in large - size pieces, and if we here reach the head of the list, the - list cannot be longer than 2000 000 undo logs now. */ - - if (trx_sys->rseg_history_len > 2000000) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: purge reached the" - " head of the history list,\n" - "InnoDB: but its length is still" - " reported as %lu! Make a detailed bug\n" - "InnoDB: report, and submit it" - " to https://jira.mariadb.org/\n", - (ulong) trx_sys->rseg_history_len); - ut_ad(0); - } - - mutex_exit(&kernel_mutex); - return; } diff --git a/storage/xtradb/trx/trx0purge.c b/storage/xtradb/trx/trx0purge.c index bc094df0364..2a9e80fe66a 100644 --- a/storage/xtradb/trx/trx0purge.c +++ b/storage/xtradb/trx/trx0purge.c @@ -728,32 +728,6 @@ trx_purge_rseg_get_next_history_log( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - - mutex_enter(&kernel_mutex); - - /* Add debug code to track history list corruption reported - on the MySQL mailing list on Nov 9, 2004. The fut0lst.c - file-based list was corrupt. The prev node pointer was - FIL_NULL, even though the list length was over 8 million nodes! - We assume that purge truncates the history list in large - size pieces, and if we here reach the head of the list, the - list cannot be longer than 2000 000 undo logs now. */ - - if (trx_sys->rseg_history_len > 2000000) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: purge reached the" - " head of the history list,\n" - "InnoDB: but its length is still" - " reported as %lu! Make a detailed bug\n" - "InnoDB: report, and submit it" - " to https://jira.mariadb.org/\n", - (ulong) trx_sys->rseg_history_len); - ut_ad(0); - } - - mutex_exit(&kernel_mutex); - return; } From 380069c235cf63cedff1855314766a545fb26846 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 13 Dec 2017 15:40:41 +0400 Subject: [PATCH 28/82] MDEV-14638 - Replace trx_sys_t::rw_trx_set with LF_HASH trx_sys_t::rw_trx_set is implemented as std::set, which does a few quite expensive operations under trx_sys_t::mutex protection: e.g. malloc/free when adding/removing elements. Traversing b-tree is not that cheap either. This has negative scalability impact, which is especially visible when running oltp_update_index.lua benchmark on a ramdisk. To reduce trx_sys_t::mutex contention std::set is replaced with LF_HASH. None of LF_HASH operations require trx_sys_t::mutex (nor any other global mutex) protection. Another interesting issue observed with std::set is reproducible ~2% performance decline after benchmark is ran for ~60 seconds. With LF_HASH results are stable. All in all this patch optimises away one of three trx_sys->mutex locks per oltp_update_index.lua query. The other two critical sections became smaller. Relevant clean-ups: Replaced rw_trx_set iteration at startup with local set. The latter is needed because values inserted to rw_trx_list must be ordered by trx->id. Removed redundant conditions from trx_reference(): it is (and even was) never called with transactions that have trx->state == TRX_STATE_COMMITTED_IN_MEMORY. do_ref_count doesn't (and probably even didn't) make any sense: now it is called only when reference counter increment is actually requested. Moved condition out of mutex in trx_erase_lists(). trx_rw_is_active(), trx_rw_is_active_low() and trx_get_rw_trx_by_id() were greatly simplified and replaced by appropriate trx_rw_hash_t methods. Compared to rw_trx_set, rw_trx_hash holds transactions only in PREPARED or ACTIVE states. Transactions in COMMITTED state were required to be found at InnoDB startup only. They are now looked up in the local set. Removed unused trx_assert_recovered(). Removed unused innobase_get_trx() declaration. Removed rather semantically incorrect trx_sys_rw_trx_add(). Moved information printout from trx_sys_init_at_db_start() to trx_lists_init_at_db_start(). --- storage/innobase/btr/btr0cur.cc | 6 +- storage/innobase/buf/buf0buf.cc | 3 - storage/innobase/handler/ha_innodb.cc | 14 +- storage/innobase/include/row0vers.h | 3 + storage/innobase/include/sync0sync.h | 1 + storage/innobase/include/sync0types.h | 2 + storage/innobase/include/trx0sys.h | 337 ++++++++++++++++++++++---- storage/innobase/include/trx0sys.ic | 142 ----------- storage/innobase/include/trx0trx.h | 20 +- storage/innobase/include/trx0trx.ic | 32 +-- storage/innobase/include/trx0types.h | 50 ---- storage/innobase/lock/lock0lock.cc | 61 ++--- storage/innobase/read/read0read.cc | 23 +- storage/innobase/row/row0row.cc | 4 +- storage/innobase/row/row0sel.cc | 10 +- storage/innobase/row/row0vers.cc | 38 ++- storage/innobase/sync/sync0debug.cc | 4 + storage/innobase/sync/sync0sync.cc | 1 + storage/innobase/trx/trx0sys.cc | 45 +--- storage/innobase/trx/trx0trx.cc | 185 ++++++++------ 20 files changed, 512 insertions(+), 469 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index fc13537d364..43163058418 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -482,10 +482,10 @@ inconsistent: /* In fact, because we only ever append fields to the 'default value' record, it is also OK to perform READ UNCOMMITTED and then ignore any extra fields, provided that - trx_rw_is_active(DB_TRX_ID). */ + trx_sys->rw_trx_hash.find(DB_TRX_ID). */ if (rec_offs_n_fields(offsets) > index->n_fields - && !trx_rw_is_active(row_get_rec_trx_id(rec, index, offsets), - NULL, false)) { + && !trx_sys->rw_trx_hash.find(row_get_rec_trx_id(rec, index, + offsets))) { goto inconsistent; } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index df47a49152c..93aa0a646d0 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -443,9 +443,6 @@ bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) MY_ATTRIBUTE((nonnull)); -/* prototypes for new functions added to ha_innodb.cc */ -trx_t* innobase_get_trx(); - /********************************************************************//** Gets the smallest oldest_modification lsn for any page in the pool. Returns zero if all modified pages have been flushed to disk. diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 53aab576202..b01eb0463f1 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2819,13 +2819,17 @@ check_trx_exists( return(trx); } -/************************************************************************* -Gets current trx. */ -trx_t* -innobase_get_trx() +/** + Gets current trx. + + This function may be called during InnoDB initialisation, when + innodb_hton_ptr->slot is not yet set to meaningful value. +*/ + +trx_t *current_trx() { THD *thd=current_thd; - if (likely(thd != 0)) { + if (likely(thd != 0) && innodb_hton_ptr->slot != HA_SLOT_UNDEF) { trx_t*& trx = thd_to_trx(thd); return(trx); } else { diff --git a/storage/innobase/include/row0vers.h b/storage/innobase/include/row0vers.h index 576b53358f8..ee0b1528683 100644 --- a/storage/innobase/include/row0vers.h +++ b/storage/innobase/include/row0vers.h @@ -41,6 +41,7 @@ class ReadView; /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) @@ -48,6 +49,7 @@ index record. @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( + trx_t* caller_trx, const rec_t* rec, dict_index_t* index, const ulint* offsets); @@ -126,6 +128,7 @@ which should be seen by a semi-consistent read. */ void row_vers_build_for_semi_consistent_read( /*====================================*/ + trx_t* caller_trx,/*!state to @@ -206,41 +198,6 @@ UNIV_INLINE trx_id_t trx_rw_min_trx_id(void); /*===================*/ -/****************************************************************//** -Checks if a rw transaction with the given id is active. -@return transaction instance if active, or NULL */ -UNIV_INLINE -trx_t* -trx_rw_is_active_low( -/*=================*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt); /*!< in: NULL or pointer to a flag - that will be set if corrupt */ -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding trx_sys->mutex, the transaction may already have been -committed. -@return transaction instance if active, or NULL; */ -UNIV_INLINE -trx_t* -trx_rw_is_active( -/*=============*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt, /*!< in: NULL or pointer to a flag - that will be set if corrupt */ - bool do_ref_count); /*!< in: if true then increment the - trx_t::n_ref_count */ -#if defined UNIV_DEBUG || defined UNIV_BLOB_LIGHT_DEBUG -/***********************************************************//** -Assert that a transaction has been recovered. -@return TRUE */ -UNIV_INLINE -ibool -trx_assert_recovered( -/*=================*/ - trx_id_t trx_id) /*!< in: transaction identifier */ - MY_ATTRIBUTE((warn_unused_result)); -#endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ /*****************************************************************//** Updates the offset information about the end of the MySQL binlog entry which corresponds to the transaction just being committed. In a MySQL @@ -302,13 +259,6 @@ ulint trx_sys_any_active_transactions(void); /*=================================*/ -/** -Add the transaction to the RW transaction set -@param trx transaction instance to add */ -UNIV_INLINE -void -trx_sys_rw_trx_add(trx_t* trx); - #ifdef UNIV_DEBUG /*************************************************************//** Validate the trx_sys_t::rw_trx_list. @@ -497,6 +447,281 @@ FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID. */ #define TRX_SYS_DOUBLEWRITE_BLOCK_SIZE FSP_EXTENT_SIZE /* @} */ +trx_t* current_trx(); + +struct rw_trx_hash_element_t +{ + rw_trx_hash_element_t(): trx(0) + { + mutex_create(LATCH_ID_RW_TRX_HASH_ELEMENT, &mutex); + } + + + ~rw_trx_hash_element_t() + { + mutex_free(&mutex); + } + + + trx_id_t id; /* lf_hash_init() relies on this to be first in the struct */ + trx_t *trx; + ib_mutex_t mutex; +}; + + +/** + Wrapper around LF_HASH to store set of in memory read-write transactions. +*/ + +class rw_trx_hash_t +{ + LF_HASH hash; + + + /** + Constructor callback for lock-free allocator. + + Object is just allocated and is not yet accessible via rw_trx_hash by + concurrent threads. Object can be reused multiple times before it is freed. + Every time object is being reused initializer() callback is called. + */ + + static void rw_trx_hash_constructor(uchar *arg) + { + new(arg + LF_HASH_OVERHEAD) rw_trx_hash_element_t(); + } + + + /** + Destructor callback for lock-free allocator. + + Object is about to be freed and is not accessible via rw_trx_hash by + concurrent threads. + */ + + static void rw_trx_hash_destructor(uchar *arg) + { + reinterpret_cast + (arg + LF_HASH_OVERHEAD)->~rw_trx_hash_element_t(); + } + + + /** + Initializer callback for lock-free hash. + + Object is not yet accessible via rw_trx_hash by concurrent threads, but is + about to become such. Object id can be changed only by this callback and + remains the same until all pins to this object are released. + + Object trx can be changed to 0 by erase() under object mutex protection, + which indicates it is about to be removed from lock-free hash and become + not accessible by concurrent threads. + */ + + static void rw_trx_hash_initializer(LF_HASH *, + rw_trx_hash_element_t *element, + trx_t *trx) + { + element->trx= trx; + element->id= trx->id; + trx->rw_trx_hash_element= element; + } + + + /** + Gets LF_HASH pins. + + Pins are used to protect object from being destroyed or reused. They are + normally stored in trx object for quick access. If caller doesn't have trx + available, we try to get it using currnet_trx(). If caller doesn't have trx + at all, temporary pins are allocated. + */ + + LF_PINS *get_pins(trx_t *trx) + { + if (!trx->rw_trx_hash_pins) + { + trx->rw_trx_hash_pins= lf_hash_get_pins(&hash); + ut_a(trx->rw_trx_hash_pins); + } + return trx->rw_trx_hash_pins; + } + + +public: + void init() + { + lf_hash_init(&hash, sizeof(rw_trx_hash_element_t), LF_HASH_UNIQUE, 0, + sizeof(trx_id_t), 0, &my_charset_bin); + hash.alloc.constructor= rw_trx_hash_constructor; + hash.alloc.destructor= rw_trx_hash_destructor; + hash.initializer= + reinterpret_cast(rw_trx_hash_initializer); + } + + + void destroy() + { + lf_hash_destroy(&hash); + } + + + /** + Releases LF_HASH pins. + + Must be called by thread that owns trx_t object when the latter is being + "detached" from thread (e.g. released to the pool by trx_free()). Can be + called earlier if thread is expected not to use rw_trx_hash. + + Since pins are not allowed to be transferred to another thread, + initialisation thread calls this for recovered transactions. + */ + + void put_pins(trx_t *trx) + { + if (trx->rw_trx_hash_pins) + { + lf_hash_put_pins(trx->rw_trx_hash_pins); + trx->rw_trx_hash_pins= 0; + } + } + + + /** + Finds trx object in lock-free hash with given id. + + Only ACTIVE or PREPARED trx objects may participate in hash. Nevertheless + the transaction may get committed before this method returns. + + With do_ref_count == false the caller may dereference returned trx pointer + only if lock_sys->mutex was acquired before calling find(). + + With do_ref_count == true caller may dereference trx even if it is not + holding lock_sys->mutex. Caller is responsible for calling + trx_release_reference() when it is done playing with trx. + + Ideally this method should get caller rw_trx_hash_pins along with trx + object as a parameter, similar to insert() and erase(). However most + callers lose trx early in their call chains and it is not that easy to pass + them through. + + So we take more expensive approach: get trx through current_thd()->ha_data. + Some threads don't have trx attached to THD, and at least server + initialisation thread, fts_optimize_thread, srv_master_thread, + dict_stats_thread, srv_monitor_thread, btr_defragment_thread don't even + have THD at all. For such cases we allocate pins only for duration of + search and free them immediately. + + This has negative performance impact and should be fixed eventually (by + passing caller_trx as a parameter). Still stream of DML is more or less Ok. + + @return + @retval 0 not found + @retval pointer to trx + */ + + trx_t *find(trx_t *caller_trx, trx_id_t trx_id, bool do_ref_count= false) + { + /* + In MariaDB 10.3, purge will reset DB_TRX_ID to 0 + when the history is lost. Read/write transactions will + always have a nonzero trx_t::id; there the value 0 is + reserved for transactions that did not write or lock + anything yet. + */ + if (!trx_id) + return NULL; + + trx_t *trx= 0; + LF_PINS *pins= caller_trx ? get_pins(caller_trx) : lf_hash_get_pins(&hash); + ut_a(pins); + + rw_trx_hash_element_t *element= reinterpret_cast + (lf_hash_search(&hash, pins, reinterpret_cast(&trx_id), + sizeof(trx_id_t))); + if (element) + { + mutex_enter(&element->mutex); + lf_hash_search_unpin(pins); + if ((trx= element->trx)) + { + if (do_ref_count) + trx_reference(trx); +#ifdef UNIV_DEBUG + mutex_enter(&trx->mutex); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + mutex_exit(&trx->mutex); +#endif + } + mutex_exit(&element->mutex); + } + if (!caller_trx) + lf_hash_put_pins(pins); + return trx; + } + + + trx_t *find(trx_id_t trx_id, bool do_ref_count= false) + { + return find(current_trx(), trx_id, do_ref_count); + } + + + /** + Inserts trx to lock-free hash. + + Object becomes accessible via rw_trx_hash. + */ + + void insert(trx_t *trx) + { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + int res= lf_hash_insert(&hash, get_pins(trx), + reinterpret_cast(trx)); + ut_a(res == 0); + } + + + /** + Removes trx from lock-free hash. + + Object becomes not accessible via rw_trx_hash. But it still can be pinned + by concurrent find(), which is supposed to release it immediately after + it sees object trx is 0. + */ + + void erase(trx_t *trx) + { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + mutex_enter(&trx->rw_trx_hash_element->mutex); + trx->rw_trx_hash_element->trx= 0; + mutex_exit(&trx->rw_trx_hash_element->mutex); + int res= lf_hash_delete(&hash, get_pins(trx), + reinterpret_cast(&trx->id), + sizeof(trx_id_t)); + ut_a(res == 0); + } + + + /** + Returns the number of elements in the hash. + + The number is exact only if hash is protected against concurrent + modifications (e.g. single threaded startup or hash is protected + by some mutex). Otherwise the number may be used as a hint only, + because it may change even before this method returns. + */ + + int32_t size() + { + return my_atomic_load32_explicit(&hash.count, MY_MEMORY_ORDER_RELAXED); + } +}; + + /** The transaction system central memory data structure. */ struct trx_sys_t { @@ -569,8 +794,16 @@ struct trx_sys_t { transactions), protected by rseg->mutex */ - TrxIdSet rw_trx_set; /*!< Mapping from transaction id - to transaction instance */ + const char rw_trx_hash_pre_pad[CACHE_LINE_SIZE]; + + + /** + Lock-free hash of in memory read-write transactions. + Works faster when it is on it's own cache line (tested). + */ + + rw_trx_hash_t rw_trx_hash; + const char rw_trx_hash_post_pad[CACHE_LINE_SIZE]; ulint n_prepared_trx; /*!< Number of transactions currently in the XA PREPARED state */ diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic index 020a46fb028..5c2a913554a 100644 --- a/storage/innobase/include/trx0sys.ic +++ b/storage/innobase/include/trx0sys.ic @@ -192,32 +192,6 @@ trx_write_trx_id( mach_write_to_6(ptr, id); } -/****************************************************************//** -Looks for the trx handle with the given id in rw_trx_list. -The caller must be holding trx_sys->mutex. -@return the trx handle or NULL if not found; -the pointer must not be dereferenced unless lock_sys->mutex was -acquired before calling this function and is still being held */ -UNIV_INLINE -trx_t* -trx_get_rw_trx_by_id( -/*=================*/ - trx_id_t trx_id) /*!< in: trx id to search for */ -{ - ut_ad(trx_id > 0); - ut_ad(trx_sys_mutex_own()); - - if (trx_sys->rw_trx_set.empty()) { - return(NULL); - } - - TrxIdSet::iterator it; - - it = trx_sys->rw_trx_set.find(TrxTrack(trx_id)); - - return(it == trx_sys->rw_trx_set.end() ? NULL : it->m_trx); -} - /****************************************************************//** Returns the minimum trx id in trx list. This is the smallest id for which the trx can possibly be active. (But, you must look at the trx->state @@ -245,29 +219,6 @@ trx_rw_min_trx_id_low(void) return(id); } -#if defined UNIV_DEBUG || defined UNIV_BLOB_LIGHT_DEBUG -/***********************************************************//** -Assert that a transaction has been recovered. -@return TRUE */ -UNIV_INLINE -ibool -trx_assert_recovered( -/*=================*/ - trx_id_t trx_id) /*!< in: transaction identifier */ -{ - const trx_t* trx; - - trx_sys_mutex_enter(); - - trx = trx_get_rw_trx_by_id(trx_id); - ut_a(trx->is_recovered); - - trx_sys_mutex_exit(); - - return(TRUE); -} -#endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ - /****************************************************************//** Returns the minimum trx id in rw trx list. This is the smallest id for which the rw trx can possibly be active. (But, you must look at the trx->state @@ -288,86 +239,6 @@ trx_rw_min_trx_id(void) return(id); } -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding lock_sys->mutex, the transaction may already have been committed. -@return transaction instance if active, or NULL */ -UNIV_INLINE -trx_t* -trx_rw_is_active_low( -/*=================*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt) /*!< in: NULL or pointer to a flag - that will be set if corrupt */ -{ - trx_t* trx; - - ut_ad(trx_sys_mutex_own()); - - if (trx_id < trx_rw_min_trx_id_low()) { - - trx = NULL; - } else if (trx_id >= trx_sys->max_trx_id) { - - /* There must be corruption: we let the caller handle the - diagnostic prints in this case. */ - - trx = NULL; - if (corrupt != NULL) { - *corrupt = TRUE; - } - } else { - trx = trx_get_rw_trx_by_id(trx_id); - - if (trx != NULL - && trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) { - - trx = NULL; - } - } - - return(trx); -} - -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding lock_sys->mutex, the transaction may already have been -committed. -@return transaction instance if active, or NULL; */ -UNIV_INLINE -trx_t* -trx_rw_is_active( -/*=============*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt, /*!< in: NULL or pointer to a flag - that will be set if corrupt */ - bool do_ref_count) /*!< in: if true then increment the - trx_t::n_ref_count */ -{ - if (!trx_id) { - /* In MariaDB 10.3, purge will reset DB_TRX_ID to 0 - when the history is lost. Read/write transactions will - always have a nonzero trx_t::id; there the value 0 is - reserved for transactions that did not write or lock - anything yet. */ - return NULL; - } - - trx_t* trx; - - trx_sys_mutex_enter(); - - trx = trx_rw_is_active_low(trx_id, corrupt); - - if (trx != 0) { - trx = trx_reference(trx, do_ref_count); - } - - trx_sys_mutex_exit(); - - return(trx); -} - /*****************************************************************//** Allocates a new transaction id. @return new, allocated trx id */ @@ -441,16 +312,3 @@ trx_sys_get_n_rw_trx(void) return(n_trx); } - -/** -Add the transaction to the RW transaction set -@param trx transaction instance to add */ -UNIV_INLINE -void -trx_sys_rw_trx_add(trx_t* trx) -{ - ut_ad(trx->id != 0); - - trx_sys->rw_trx_set.insert(TrxTrack(trx->id, trx)); - ut_d(trx->in_rw_trx_list = true); -} diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 81ef6093e2e..9366d20b1f5 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -55,6 +55,8 @@ class ReadView; // Forward declaration class FlushObserver; +struct rw_trx_hash_element_t; + /** Dummy session used currently in MySQL interface */ extern sess_t* trx_dummy_sess; @@ -531,17 +533,12 @@ trx_set_rw_mode( trx_t* trx); /** -Increase the reference count. If the transaction is in state -TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered -committed and the reference count is not incremented. -@param trx Transaction that is being referenced -@param do_ref_count Increment the reference iff this is true -@return transaction instance if it is not committed */ +Increase the reference count. +@param trx Transaction that is being referenced */ UNIV_INLINE -trx_t* +void trx_reference( - trx_t* trx, - bool do_ref_count); + trx_t* trx); /** Release the transaction. Decrease the reference count. @@ -951,6 +948,9 @@ struct trx_t { Recovered XA: * NOT_STARTED -> PREPARED -> COMMITTED -> (freed) + Recovered XA followed by XA ROLLBACK: + * NOT_STARTED -> PREPARED -> ACTIVE -> COMMITTED -> (freed) + XA (2PC) (shutdown or disconnect before ROLLBACK or COMMIT): * NOT_STARTED -> PREPARED -> (freed) @@ -1277,6 +1277,8 @@ struct trx_t { os_event_t wsrep_event; /* event waited for in srv_conc_slot */ #endif /* WITH_WSREP */ + rw_trx_hash_element_t *rw_trx_hash_element; + LF_PINS *rw_trx_hash_pins; ulint magic_n; /** @return whether any persistent undo log has been generated */ diff --git a/storage/innobase/include/trx0trx.ic b/storage/innobase/include/trx0trx.ic index 6fa00c5333f..cd0b812869d 100644 --- a/storage/innobase/include/trx0trx.ic +++ b/storage/innobase/include/trx0trx.ic @@ -214,32 +214,14 @@ ok: } /** -Increase the reference count. If the transaction is in state -TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered -committed and the reference count is not incremented. -@param trx Transaction that is being referenced -@param do_ref_count Increment the reference iff this is true -@return transaction instance if it is not committed */ -UNIV_INLINE -trx_t* -trx_reference( - trx_t* trx, - bool do_ref_count) +Increase the reference count. +@param trx Transaction that is being referenced */ +UNIV_INLINE void trx_reference(trx_t *trx) { - trx_mutex_enter(trx); - - if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) { - trx_mutex_exit(trx); - trx = NULL; - } else if (do_ref_count) { - ut_ad(trx->n_ref >= 0); - ++trx->n_ref; - trx_mutex_exit(trx); - } else { - trx_mutex_exit(trx); - } - - return(trx); + trx_mutex_enter(trx); + ut_ad(trx->n_ref >= 0); + ++trx->n_ref; + trx_mutex_exit(trx); } /** diff --git a/storage/innobase/include/trx0types.h b/storage/innobase/include/trx0types.h index 8092246c7fa..2b080eea2d3 100644 --- a/storage/innobase/include/trx0types.h +++ b/storage/innobase/include/trx0types.h @@ -31,12 +31,9 @@ Created 3/26/1996 Heikki Tuuri #include "ut0mutex.h" #include "ut0new.h" -#include #include #include -//#include - /** printf(3) format used for printing DB_TRX_ID and other system fields */ #define TRX_ID_FMT IB_ID_FMT @@ -173,51 +170,4 @@ typedef ib_mutex_t PQMutex; typedef ib_mutex_t TrxSysMutex; typedef std::vector > trx_ids_t; - -/** Mapping read-write transactions from id to transaction instance, for -creating read views and during trx id lookup for MVCC and locking. */ -struct TrxTrack { - explicit TrxTrack(trx_id_t id, trx_t* trx = NULL) - : - m_id(id), - m_trx(trx) - { - // Do nothing - } - - trx_id_t m_id; - trx_t* m_trx; -}; - -struct TrxTrackHash { - size_t operator()(const TrxTrack& key) const - { - return(size_t(key.m_id)); - } -}; - -/** -Comparator for TrxMap */ -struct TrxTrackHashCmp { - - bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const - { - return(lhs.m_id == rhs.m_id); - } -}; - -/** -Comparator for TrxMap */ -struct TrxTrackCmp { - - bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const - { - return(lhs.m_id < rhs.m_id); - } -}; - -//typedef std::unordered_set TrxIdSet; -typedef std::set > - TrxIdSet; - #endif /* trx0types_h */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 2f66e178daf..269f606250f 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1494,6 +1494,7 @@ static trx_t* lock_sec_rec_some_has_impl( /*=======================*/ + trx_t* caller_trx,/*!mutex); - if (trx_t* impl_trx = trx_rw_is_active(trx->id, NULL, false)) { - ulint heap_no = page_rec_get_heap_no(rec); - mutex_enter(&trx_sys->mutex); + for (trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); + t != NULL; + t = UT_LIST_GET_NEXT(trx_list, t)) { - for (trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); - t != NULL; - t = UT_LIST_GET_NEXT(trx_list, t)) { + lock_t* expl_lock = lock_rec_has_expl( + precise_mode, block, heap_no, t); - lock_t* expl_lock = lock_rec_has_expl( - precise_mode, block, heap_no, t); - - if (expl_lock && expl_lock->trx != impl_trx) { - /* An explicit lock is held by trx other than - the trx holding the implicit lock. */ - holds = expl_lock->trx; - break; - } + if (expl_lock && expl_lock->trx != trx) { + /* An explicit lock is held by trx other than + the trx holding the implicit lock. */ + holds = expl_lock->trx; + break; } - - mutex_exit(&trx_sys->mutex); } + mutex_exit(&trx_sys->mutex); + lock_mutex_exit(); return(holds); @@ -6223,7 +6226,6 @@ lock_rec_queue_validate( const dict_index_t* index, /*!< in: index, or NULL if not known */ const ulint* offsets)/*!< in: rec_get_offsets(rec, index) */ { - const trx_t* impl_trx; const lock_t* lock; ulint heap_no; @@ -6269,13 +6271,11 @@ lock_rec_queue_validate( /* Nothing we can do */ } else if (dict_index_is_clust(index)) { - trx_id_t trx_id; - /* Unlike the non-debug code, this invariant can only succeed if the check and assertion are covered by the lock mutex. */ - trx_id = lock_clust_rec_some_has_impl(rec, index, offsets); - impl_trx = trx_rw_is_active_low(trx_id, NULL); + const trx_t *impl_trx = trx_sys->rw_trx_hash.find( + lock_clust_rec_some_has_impl(rec, index, offsets)); ut_ad(lock_mutex_own()); /* impl_trx cannot be committed until lock_mutex_exit() @@ -6849,6 +6849,7 @@ static void lock_rec_convert_impl_to_expl( /*==========================*/ + trx_t* caller_trx,/*!rw_trx_hash.find(caller_trx, trx_id, true); } else { ut_ad(!dict_index_is_online_ddl(index)); - trx = lock_sec_rec_some_has_impl(rec, index, offsets); + trx = lock_sec_rec_some_has_impl(caller_trx, rec, index, + offsets); ut_ad(!trx || !lock_rec_other_trx_holds_expl( LOCK_S | LOCK_REC_NOT_GAP, trx, rec, block)); @@ -6934,7 +6936,8 @@ lock_clust_rec_modify_check_and_lock( /* If a transaction has no explicit x-lock set on the record, set one for it */ - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, index, + offsets); lock_mutex_enter(); @@ -7098,7 +7101,8 @@ lock_sec_rec_read_check_and_lock( || recv_recovery_is_on()) && !page_rec_is_supremum(rec)) { - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, + index, offsets); } lock_mutex_enter(); @@ -7173,7 +7177,8 @@ lock_clust_rec_read_check_and_lock( if (heap_no != PAGE_HEAP_NO_SUPREMUM) { - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, + index, offsets); } lock_mutex_enter(); diff --git a/storage/innobase/read/read0read.cc b/storage/innobase/read/read0read.cc index 2fb7083b0b2..8f4dd4f37b2 100644 --- a/storage/innobase/read/read0read.cc +++ b/storage/innobase/read/read0read.cc @@ -371,6 +371,7 @@ Copy the transaction ids from the source vector */ void ReadView::copy_trx_ids(const trx_ids_t& trx_ids) { + ut_ad(mutex_own(&trx_sys->mutex)); ulint size = trx_ids.size(); if (m_creator_trx_id > 0) { @@ -424,14 +425,24 @@ ReadView::copy_trx_ids(const trx_ids_t& trx_ids) } #ifdef UNIV_DEBUG - /* Assert that all transaction ids in list are active. */ + /* Original assertion was here to make sure that rw_trx_ids and + rw_trx_hash are in sync and they hold either ACTIVE or PREPARED + transaction. + + Now rw_trx_hash.find() does + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)). + No need to repeat it here. We even can't repeat it here: it'll be race + condition because we need trx->element->mutex locked to perform this + check (see how it is done in find()). + + Now rw_trx_ids and rw_trx_hash may get out of sync for a short while: + when transaction is registered it first gets added into rw_trx_ids + under trx_sys->mutex protection and then to rw_trx_hash without mutex + protection. Thus we need repeat this lookup. */ for (trx_ids_t::const_iterator it = trx_ids.begin(); it != trx_ids.end(); ++it) { - - trx_t* trx = trx_get_rw_trx_by_id(*it); - ut_ad(trx != NULL); - ut_ad(trx->state == TRX_STATE_ACTIVE - || trx->state == TRX_STATE_PREPARED); + while (!trx_sys->rw_trx_hash.find(*it)); } #endif /* UNIV_DEBUG */ } diff --git a/storage/innobase/row/row0row.cc b/storage/innobase/row/row0row.cc index 3611b8c3fc7..3f6c8b9f918 100644 --- a/storage/innobase/row/row0row.cc +++ b/storage/innobase/row/row0row.cc @@ -415,8 +415,8 @@ row_build_low( times, and the cursor restore can happen multiple times for single insert or update statement. */ ut_a(!rec_offs_any_null_extern(rec, offsets) - || trx_rw_is_active(row_get_rec_trx_id(rec, index, offsets), - NULL, false)); + || trx_sys->rw_trx_hash.find(row_get_rec_trx_id(rec, index, + offsets))); #endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ if (type != ROW_COPY_POINTERS) { diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 53f60f2f70b..eeb52bfeee3 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -799,7 +799,7 @@ row_sel_build_committed_vers_for_mysql( rec_offs_size(*offsets)); } - row_vers_build_for_semi_consistent_read( + row_vers_build_for_semi_consistent_read(prebuilt->trx, rec, mtr, clust_index, offsets, offset_heap, prebuilt->old_vers_heap, old_vers, vrow); } @@ -4972,17 +4972,17 @@ wrong_offs: /* In delete-marked records, DB_TRX_ID must always refer to an existing undo log record. */ ut_ad(trx_id); - if (!trx_rw_is_active(trx_id, NULL, false)) { + if (!trx_sys->rw_trx_hash.find(trx, trx_id)) { /* The clustered index record was delete-marked in a committed transaction. Ignore the record. */ goto locks_ok_del_marked; } - } else if (trx_t* trx = row_vers_impl_x_locked( - rec, index, offsets)) { + } else if (trx_t* t = row_vers_impl_x_locked( + trx, rec, index, offsets)) { /* The record belongs to an active transaction. We must acquire a lock. */ - trx_release_reference(trx); + trx_release_reference(t); } else { /* The secondary index record does not point to a delete-marked clustered index diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 712c46cba35..0b8cf63f940 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -47,6 +47,7 @@ Created 2/6/1997 Heikki Tuuri /** Check whether all non-virtual columns in a virtual index match that of in the cluster index +@param[in,out] caller_trx trx of current thread @param[in] index the secondary index @param[in] row the cluster index row in dtuple form @param[in] ext externally stored column prefix or NULL @@ -65,6 +66,7 @@ row_vers_non_vc_match( ulint* n_non_v_col); /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] clust_rec clustered index record @param[in] clust_index clustered index @param[in] rec secondary index record @@ -76,6 +78,7 @@ index record. UNIV_INLINE trx_t* row_vers_impl_x_locked_low( + trx_t* caller_trx, const rec_t* clust_rec, dict_index_t* clust_index, const rec_t* rec, @@ -84,7 +87,6 @@ row_vers_impl_x_locked_low( mtr_t* mtr) { trx_id_t trx_id; - ibool corrupt; ulint comp; ulint rec_del; const rec_t* version; @@ -118,13 +120,15 @@ row_vers_impl_x_locked_low( mem_heap_free(heap); DBUG_RETURN(0); } - corrupt = FALSE; - trx_t* trx = trx_rw_is_active(trx_id, &corrupt, true); + trx_t* trx = trx_sys->rw_trx_hash.find(caller_trx, trx_id, true); if (trx == 0) { /* The transaction that modified or inserted clust_rec is no longer active, or it is corrupt: no implicit lock on rec */ + trx_sys_mutex_enter(); + bool corrupt = trx_id >= trx_sys->max_trx_id; + trx_sys_mutex_exit(); if (corrupt) { lock_report_trx_id_insanity( trx_id, clust_rec, clust_index, clust_offsets, @@ -189,7 +193,7 @@ row_vers_impl_x_locked_low( inserting a delete-marked record. */ ut_ad(prev_version || !rec_get_deleted_flag(version, comp) - || !trx_rw_is_active(trx_id, NULL, false)); + || !trx_sys->rw_trx_hash.find(caller_trx, trx_id)); /* Free version and clust_offsets. */ mem_heap_free(old_heap); @@ -342,6 +346,7 @@ result_check: /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) @@ -349,6 +354,7 @@ index record. @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( + trx_t* caller_trx, const rec_t* rec, dict_index_t* index, const ulint* offsets) @@ -389,7 +395,8 @@ row_vers_impl_x_locked( trx = 0; } else { trx = row_vers_impl_x_locked_low( - clust_rec, clust_index, rec, index, offsets, &mtr); + caller_trx, clust_rec, clust_index, rec, index, + offsets, &mtr); ut_ad(trx == 0 || trx_is_referenced(trx)); } @@ -1234,6 +1241,7 @@ which should be seen by a semi-consistent read. */ void row_vers_build_for_semi_consistent_read( /*====================================*/ + trx_t* caller_trx,/*!mutex. It may change from - ACTIVE to PREPARED or COMMITTED. */ - if (version_trx - && trx_state_eq(version_trx, - TRX_STATE_COMMITTED_IN_MEMORY)) { - version_trx = NULL; - } - trx_sys_mutex_exit(); - - if (!version_trx) { + if (!trx_sys->rw_trx_hash.find(caller_trx, version_trx_id)) { committed_version_trx: /* We found a version that belongs to a committed transaction: return it. */ diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 76796f96913..e61c01c0443 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -480,6 +480,7 @@ LatchDebug::LatchDebug() LEVEL_MAP_INSERT(SYNC_REC_LOCK); LEVEL_MAP_INSERT(SYNC_THREADS); LEVEL_MAP_INSERT(SYNC_TRX); + LEVEL_MAP_INSERT(SYNC_RW_TRX_HASH_ELEMENT); LEVEL_MAP_INSERT(SYNC_TRX_SYS); LEVEL_MAP_INSERT(SYNC_LOCK_SYS); LEVEL_MAP_INSERT(SYNC_LOCK_WAIT_SYS); @@ -761,6 +762,7 @@ LatchDebug::check_order( case SYNC_THREADS: case SYNC_LOCK_SYS: case SYNC_LOCK_WAIT_SYS: + case SYNC_RW_TRX_HASH_ELEMENT: case SYNC_TRX_SYS: case SYNC_IBUF_BITMAP_MUTEX: case SYNC_REDO_RSEG: @@ -1521,6 +1523,8 @@ sync_latch_meta_init() PFS_NOT_INSTRUMENTED); LATCH_ADD_MUTEX(FIL_CRYPT_THREADS_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(RW_TRX_HASH_ELEMENT, SYNC_RW_TRX_HASH_ELEMENT, + rw_trx_hash_element_mutex_key); latch_id_t id = LATCH_ID_NONE; diff --git a/storage/innobase/sync/sync0sync.cc b/storage/innobase/sync/sync0sync.cc index 5d98e49fd6d..e78b2a033ac 100644 --- a/storage/innobase/sync/sync0sync.cc +++ b/storage/innobase/sync/sync0sync.cc @@ -96,6 +96,7 @@ mysql_pfs_key_t sync_array_mutex_key; mysql_pfs_key_t thread_mutex_key; mysql_pfs_key_t zip_pad_mutex_key; mysql_pfs_key_t row_drop_list_mutex_key; +mysql_pfs_key_t rw_trx_hash_element_mutex_key; #endif /* UNIV_PFS_MUTEX */ #ifdef UNIV_PFS_RWLOCK mysql_pfs_key_t btr_search_latch_key; diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index c2e15fea953..ef69bd3102b 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -429,8 +429,6 @@ void trx_sys_init_at_db_start() { trx_sysf_t* sys_header; - ib_uint64_t rows_to_undo = 0; - const char* unit = ""; /* VERY important: after the database is started, max_trx_id value is divisible by TRX_SYS_TRX_ID_WRITE_MARGIN, and the 'if' in @@ -455,43 +453,6 @@ trx_sys_init_at_db_start() trx_dummy_sess = sess_open(); trx_lists_init_at_db_start(); - - /* This mutex is not strictly required, it is here only to satisfy - the debug code (assertions). We are still running in single threaded - bootstrap mode. */ - - trx_sys_mutex_enter(); - - if (UT_LIST_GET_LEN(trx_sys->rw_trx_list) > 0) { - const trx_t* trx; - - for (trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); - trx != NULL; - trx = UT_LIST_GET_NEXT(trx_list, trx)) { - - ut_ad(trx->is_recovered); - assert_trx_in_rw_list(trx); - - if (trx_state_eq(trx, TRX_STATE_ACTIVE)) { - rows_to_undo += trx->undo_no; - } - } - - if (rows_to_undo > 1000000000) { - unit = "M"; - rows_to_undo = rows_to_undo / 1000000; - } - - ib::info() << UT_LIST_GET_LEN(trx_sys->rw_trx_list) - << " transaction(s) which must be rolled back or" - " cleaned up in total " << rows_to_undo << unit - << " row operations to undo"; - - ib::info() << "Trx id counter is " << trx_sys->max_trx_id; - } - - trx_sys_mutex_exit(); - trx_sys->mvcc->clone_oldest_view(&purge_sys->view); } @@ -515,8 +476,7 @@ trx_sys_create(void) new(&trx_sys->rw_trx_ids) trx_ids_t(ut_allocator( mem_key_trx_sys_t_rw_trx_ids)); - - new(&trx_sys->rw_trx_set) TrxIdSet(); + trx_sys->rw_trx_hash.init(); } /*****************************************************************//** @@ -669,8 +629,7 @@ trx_sys_close(void) trx_sys->rw_trx_ids.~trx_ids_t(); - trx_sys->rw_trx_set.~TrxIdSet(); - + trx_sys->rw_trx_hash.destroy(); ut_free(trx_sys); trx_sys = NULL; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index fde7443394e..d0f79bad3a3 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -236,6 +236,7 @@ struct TrxFactory { new(&trx->hit_list) hit_list_t(); + trx->rw_trx_hash_pins = 0; trx_init(trx); DBUG_LOG("trx", "Init: " << trx); @@ -446,6 +447,7 @@ trx_create_low() /* We just got trx from pool, it should be non locking */ ut_ad(trx->will_lock == 0); + ut_ad(!trx->rw_trx_hash_pins); /* Background trx should not be forced to rollback, we will unset the flag for user trx. */ @@ -483,6 +485,7 @@ trx_free(trx_t*& trx) { assert_trx_is_free(trx); + trx_sys->rw_trx_hash.put_pins(trx); trx->mysql_thd = 0; trx->mysql_log_file_name = 0; @@ -731,7 +734,9 @@ trx_resurrect_table_locks( trx_undo_rec_t* undo_rec; table_id_set tables; - if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY) || undo->empty) { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + if (undo->empty) { return; } @@ -960,10 +965,65 @@ trx_resurrect_update( } } +/** Mapping read-write transactions from id to transaction instance, for +creating read views and during trx id lookup for MVCC and locking. */ +struct TrxTrack { + explicit TrxTrack(trx_id_t id, trx_t* trx = NULL) + : + m_id(id), + m_trx(trx) + { + // Do nothing + } + + trx_id_t m_id; + trx_t* m_trx; +}; + +/** +Comparator for TrxMap */ +struct TrxTrackCmp { + + bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const + { + return(lhs.m_id < rhs.m_id); + } +}; + +typedef std::set > + TrxIdSet; + +static inline void trx_sys_add_trx_at_init(trx_t *trx, trx_undo_t *undo, + uint64_t *rows_to_undo, + TrxIdSet *set) +{ + ut_ad(trx->id != 0); + ut_ad(trx->is_recovered); + + set->insert(TrxTrack(trx->id, trx)); + if (trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)) + { + trx_sys->rw_trx_hash.insert(trx); + trx_sys->rw_trx_hash.put_pins(trx); + trx_sys->rw_trx_ids.push_back(trx->id); + trx_resurrect_table_locks(trx, undo); + if (trx_state_eq(trx, TRX_STATE_ACTIVE)) + *rows_to_undo+= trx->undo_no; + } +#ifdef UNIV_DEBUG + trx->in_rw_trx_list= true; + if (trx->id > trx_sys->rw_max_trx_id) + trx_sys->rw_max_trx_id= trx->id; +#endif +} + /** Initialize (resurrect) transactions at startup. */ void trx_lists_init_at_db_start() { + TrxIdSet set; + uint64_t rows_to_undo = 0; ut_a(srv_is_being_started); ut_ad(!srv_was_started); ut_ad(!purge_sys); @@ -993,15 +1053,10 @@ trx_lists_init_at_db_start() for (undo = UT_LIST_GET_FIRST(rseg->old_insert_list); undo != NULL; undo = UT_LIST_GET_NEXT(undo_list, undo)) { - - trx_t* trx; - - trx = trx_resurrect_insert(undo, rseg); + trx_t* trx = trx_resurrect_insert(undo, rseg); trx->start_time = start_time; - - trx_sys_rw_trx_add(trx); - - trx_resurrect_table_locks(trx, undo); + trx_sys_add_trx_at_init(trx, undo, &rows_to_undo, + &set); } /* Ressurrect other transactions. */ @@ -1009,12 +1064,10 @@ trx_lists_init_at_db_start() undo != NULL; undo = UT_LIST_GET_NEXT(undo_list, undo)) { - /* Check the trx_sys->rw_trx_set first. */ - trx_sys_mutex_enter(); - - trx_t* trx = trx_get_rw_trx_by_id(undo->trx_id); - - trx_sys_mutex_exit(); + /* Check if trx_id was already registered first. */ + TrxIdSet::iterator it = + set.find(TrxTrack(undo->trx_id)); + trx_t *trx= it == set.end() ? 0 : it->m_trx; if (trx == NULL) { trx = trx_allocate_for_background(); @@ -1025,34 +1078,30 @@ trx_lists_init_at_db_start() } trx_resurrect_update(trx, undo, rseg); - - trx_sys_rw_trx_add(trx); - - trx_resurrect_table_locks(trx, undo); + trx_sys_add_trx_at_init(trx, undo, &rows_to_undo, + &set); } } - TrxIdSet::iterator end = trx_sys->rw_trx_set.end(); + if (set.size()) { - for (TrxIdSet::iterator it = trx_sys->rw_trx_set.begin(); + ib::info() << set.size() + << " transaction(s) which must be rolled back or" + " cleaned up in total " << rows_to_undo + << " row operations to undo"; + + ib::info() << "Trx id counter is " << trx_sys->max_trx_id; + } + + TrxIdSet::iterator end = set.end(); + + for (TrxIdSet::iterator it = set.begin(); it != end; ++it) { - ut_ad(it->m_trx->in_rw_trx_list); -#ifdef UNIV_DEBUG - if (it->m_trx->id > trx_sys->rw_max_trx_id) { - trx_sys->rw_max_trx_id = it->m_trx->id; - } -#endif /* UNIV_DEBUG */ - - if (it->m_trx->state == TRX_STATE_ACTIVE - || it->m_trx->state == TRX_STATE_PREPARED) { - - trx_sys->rw_trx_ids.push_back(it->m_id); - } - UT_LIST_ADD_FIRST(trx_sys->rw_trx_list, it->m_trx); } + std::sort(trx_sys->rw_trx_ids.begin(), trx_sys->rw_trx_ids.end()); } /** Assign a persistent rollback segment in a round-robin fashion, @@ -1171,8 +1220,8 @@ trx_t::assign_temp_rseg() mutex_enter(&trx_sys->mutex); id = trx_sys_get_new_trx_id(); trx_sys->rw_trx_ids.push_back(id); - trx_sys->rw_trx_set.insert(TrxTrack(id, this)); mutex_exit(&trx_sys->mutex); + trx_sys->rw_trx_hash.insert(this); } ut_ad(!rseg->is_persistent()); @@ -1237,10 +1286,14 @@ trx_start_low( ut_ad(!trx->in_rw_trx_list); - /* We tend to over assert and that complicates the code somewhat. - e.g., the transaction state can be set earlier but we are forced to - set it under the protection of the trx_sys_t::mutex because some - trx list assertions are triggered unnecessarily. */ + /* No other thread can access this trx object through rw_trx_hash, thus + we don't need trx_sys->mutex protection for that purpose. Still this + trx can be found through trx_sys->mysql_trx_list, which means state + change must be protected by e.g. trx->mutex. + + For now we update it without mutex protection, because original code + did it this way. It has to be reviewed and fixed properly. */ + trx->state = TRX_STATE_ACTIVE; /* By default all transactions are in the read-only list unless they are non-locking auto-commit read only transactions or background @@ -1262,8 +1315,6 @@ trx_start_low( trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys_rw_trx_add(trx); - ut_ad(trx->rsegs.m_redo.rseg != 0 || srv_read_only_mode || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); @@ -1277,11 +1328,10 @@ trx_start_low( } #endif /* UNIV_DEBUG */ - trx->state = TRX_STATE_ACTIVE; - ut_ad(trx_sys_validate_trx_list()); trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.insert(trx); } else { trx->id = 0; @@ -1302,17 +1352,11 @@ trx_start_low( trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys->rw_trx_set.insert( - TrxTrack(trx->id, trx)); - trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.insert(trx); } - - trx->state = TRX_STATE_ACTIVE; - } else { ut_ad(!read_write); - trx->state = TRX_STATE_ACTIVE; } } @@ -1642,7 +1686,22 @@ trx_erase_lists( bool serialised) { ut_ad(trx->id > 0); - trx_sys_mutex_enter(); + + if (trx->read_only || trx->rsegs.m_redo.rseg == NULL) { + + trx_sys_mutex_enter(); + ut_ad(!trx->in_rw_trx_list); + } else { + + trx_sys_mutex_enter(); + UT_LIST_REMOVE(trx_sys->rw_trx_list, trx); + ut_d(trx->in_rw_trx_list = false); + ut_ad(trx_sys_validate_trx_list()); + + if (trx->read_view != NULL) { + trx_sys->mvcc->view_close(trx->read_view, true); + } + } if (serialised) { UT_LIST_REMOVE(trx_sys->serialisation_list, trx); @@ -1654,24 +1713,8 @@ trx_erase_lists( trx->id); ut_ad(*it == trx->id); trx_sys->rw_trx_ids.erase(it); - - if (trx->read_only || trx->rsegs.m_redo.rseg == NULL) { - - ut_ad(!trx->in_rw_trx_list); - } else { - - UT_LIST_REMOVE(trx_sys->rw_trx_list, trx); - ut_d(trx->in_rw_trx_list = false); - ut_ad(trx_sys_validate_trx_list()); - - if (trx->read_view != NULL) { - trx_sys->mvcc->view_close(trx->read_view, true); - } - } - - trx_sys->rw_trx_set.erase(TrxTrack(trx->id)); - trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.erase(trx); } /****************************************************************//** @@ -3036,6 +3079,7 @@ trx_set_rw_mode( ut_ad(!trx->in_rw_trx_list); ut_ad(!trx_is_autocommit_non_locking(trx)); ut_ad(!trx->read_only); + ut_ad(trx->id == 0); if (high_level_read_only) { return; @@ -3053,14 +3097,10 @@ trx_set_rw_mode( ut_ad(trx->rsegs.m_redo.rseg != 0); mutex_enter(&trx_sys->mutex); - - ut_ad(trx->id == 0); trx->id = trx_sys_get_new_trx_id(); trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys->rw_trx_set.insert(TrxTrack(trx->id, trx)); - /* So that we can see our own changes. */ if (MVCC::is_view_active(trx->read_view)) { MVCC::set_view_creator_trx_id(trx->read_view, trx->id); @@ -3077,6 +3117,7 @@ trx_set_rw_mode( ut_d(trx->in_rw_trx_list = true); mutex_exit(&trx_sys->mutex); + trx_sys->rw_trx_hash.insert(trx); } /** From 0ca2ea1a6550bb718efc878ed451be69e8244cd4 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 20 Dec 2017 14:59:36 +0400 Subject: [PATCH 29/82] MDEV-14638 - Replace trx_sys_t::rw_trx_set with LF_HASH trx reference counter was updated under mutex and read without any protection. This is both slow and unsafe. Use atomic operations for reference counter accesses. --- storage/innobase/include/row0vers.h | 2 +- storage/innobase/include/trx0sys.h | 4 +- storage/innobase/include/trx0trx.h | 68 +++++++++++++++++------------ storage/innobase/include/trx0trx.ic | 28 ------------ storage/innobase/lock/lock0lock.cc | 12 ++--- storage/innobase/row/row0sel.cc | 2 +- storage/innobase/row/row0vers.cc | 10 ++--- 7 files changed, 55 insertions(+), 71 deletions(-) diff --git a/storage/innobase/include/row0vers.h b/storage/innobase/include/row0vers.h index ee0b1528683..645f11faaad 100644 --- a/storage/innobase/include/row0vers.h +++ b/storage/innobase/include/row0vers.h @@ -45,7 +45,7 @@ index record. @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index 2e87d500896..e87aa1bef6c 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -598,7 +598,7 @@ public: With do_ref_count == true caller may dereference trx even if it is not holding lock_sys->mutex. Caller is responsible for calling - trx_release_reference() when it is done playing with trx. + trx->release_reference() when it is done playing with trx. Ideally this method should get caller rw_trx_hash_pins along with trx object as a parameter, similar to insert() and erase(). However most @@ -646,7 +646,7 @@ public: if ((trx= element->trx)) { if (do_ref_count) - trx_reference(trx); + trx->reference(); #ifdef UNIV_DEBUG mutex_enter(&trx->mutex); ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 9366d20b1f5..f6a997b8a0f 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -532,26 +532,6 @@ void trx_set_rw_mode( trx_t* trx); -/** -Increase the reference count. -@param trx Transaction that is being referenced */ -UNIV_INLINE -void -trx_reference( - trx_t* trx); - -/** -Release the transaction. Decrease the reference count. -@param trx Transaction that is being released */ -UNIV_INLINE -void -trx_release_reference( - trx_t* trx); - -/** -Check if the transaction is being referenced. */ -#define trx_is_referenced(t) ((t)->n_ref > 0) - /** @param[in] requestor Transaction requesting the lock @param[in] holder Transaction holding the lock @@ -889,6 +869,19 @@ struct TrxVersion { typedef std::list > hit_list_t; struct trx_t { +private: + /** + Count of references. + + We can't release the locks nor commit the transaction until this reference + is 0. We can change the state to TRX_STATE_COMMITTED_IN_MEMORY to signify + that it is no longer "active". + */ + + int32_t n_ref; + + +public: TrxMutex mutex; /*!< Mutex protecting the fields state and lock (except some fields of lock, which are protected by @@ -1235,14 +1228,6 @@ struct trx_t { const char* start_file; /*!< Filename where it was started */ #endif /* UNIV_DEBUG */ - lint n_ref; /*!< Count of references, protected - by trx_t::mutex. We can't release the - locks nor commit the transaction until - this reference is 0. We can change - the state to COMMITTED_IN_MEMORY to - signify that it is no longer - "active". */ - /** Version of this instance. It is incremented each time the instance is re-used in trx_start_low(). It is used to track whether a transaction has been restarted since it was tagged @@ -1311,6 +1296,33 @@ struct trx_t { return(assign_temp_rseg()); } + + bool is_referenced() + { + return my_atomic_load32_explicit(&n_ref, MY_MEMORY_ORDER_RELAXED) > 0; + } + + + void reference() + { +#ifdef UNIV_DEBUG + int32_t old_n_ref= +#endif + my_atomic_add32_explicit(&n_ref, 1, MY_MEMORY_ORDER_RELAXED); + ut_ad(old_n_ref >= 0); + } + + + void release_reference() + { +#ifdef UNIV_DEBUG + int32_t old_n_ref= +#endif + my_atomic_add32_explicit(&n_ref, -1, MY_MEMORY_ORDER_RELAXED); + ut_ad(old_n_ref > 0); + } + + private: /** Assign a rollback segment for modifying temporary tables. @return the assigned rollback segment */ diff --git a/storage/innobase/include/trx0trx.ic b/storage/innobase/include/trx0trx.ic index cd0b812869d..c288da8bf0b 100644 --- a/storage/innobase/include/trx0trx.ic +++ b/storage/innobase/include/trx0trx.ic @@ -213,34 +213,6 @@ ok: trx->dict_operation = op; } -/** -Increase the reference count. -@param trx Transaction that is being referenced */ -UNIV_INLINE void trx_reference(trx_t *trx) -{ - trx_mutex_enter(trx); - ut_ad(trx->n_ref >= 0); - ++trx->n_ref; - trx_mutex_exit(trx); -} - -/** -Release the transaction. Decrease the reference count. -@param trx Transaction that is being released */ -UNIV_INLINE -void -trx_release_reference( - trx_t* trx) -{ - trx_mutex_enter(trx); - - ut_ad(trx->n_ref > 0); - --trx->n_ref; - - trx_mutex_exit(trx); -} - - /** @param trx Get the active view for this transaction, if one exists @return the transaction's read view or NULL if one not assigned. */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 269f606250f..2a6167ec809 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -6813,7 +6813,7 @@ lock_rec_convert_impl_to_expl_for_trx( trx_t* trx, /*!< in/out: active transaction */ ulint heap_no)/*!< in: rec heap number to lock */ { - ut_ad(trx_is_referenced(trx)); + ut_ad(trx->is_referenced()); ut_ad(page_rec_is_leaf(rec)); ut_ad(!rec_is_default_row(rec, index)); @@ -6837,7 +6837,7 @@ lock_rec_convert_impl_to_expl_for_trx( lock_mutex_exit(); - trx_release_reference(trx); + trx->release_reference(); DEBUG_SYNC_C("after_lock_rec_convert_impl_to_expl_for_trx"); } @@ -6883,7 +6883,7 @@ lock_rec_convert_impl_to_expl( if (trx != 0) { ulint heap_no = page_rec_get_heap_no(rec); - ut_ad(trx_is_referenced(trx)); + ut_ad(trx->is_referenced()); /* If the transaction is still active and has no explicit x-lock set on the record, set one for it. @@ -7655,13 +7655,13 @@ lock_trx_release_locks( trx->state = TRX_STATE_COMMITTED_IN_MEMORY; /*--------------------------------------*/ - if (trx_is_referenced(trx)) { + if (trx->is_referenced()) { ut_a(release_lock); lock_mutex_exit(); - while (trx_is_referenced(trx)) { + while (trx->is_referenced()) { trx_mutex_exit(trx); @@ -7681,7 +7681,7 @@ lock_trx_release_locks( trx_mutex_enter(trx); } - ut_ad(!trx_is_referenced(trx)); + ut_ad(!trx->is_referenced()); /* If the background thread trx_rollback_or_clean_recovered() is still active then there is a chance that the rollback diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index eeb52bfeee3..1395faaad96 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -4982,7 +4982,7 @@ wrong_offs: trx, rec, index, offsets)) { /* The record belongs to an active transaction. We must acquire a lock. */ - trx_release_reference(t); + t->release_reference(); } else { /* The secondary index record does not point to a delete-marked clustered index diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 0b8cf63f940..fc8b1b2ec81 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -73,7 +73,7 @@ index record. @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) @param[in,out] mtr mini-transaction -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ UNIV_INLINE trx_t* @@ -217,7 +217,7 @@ row_vers_impl_x_locked_low( or updated, the leaf page record always is created with a clear delete-mark flag. (We never insert a delete-marked record.) */ - trx_release_reference(trx); + trx->release_reference(); trx = 0; } @@ -328,7 +328,7 @@ result_check: /* prev_version was the first version modified by the trx_id transaction: no implicit x-lock */ - trx_release_reference(trx); + trx->release_reference(); trx = 0; break; } @@ -350,7 +350,7 @@ index record. @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( @@ -398,7 +398,7 @@ row_vers_impl_x_locked( caller_trx, clust_rec, clust_index, rec, index, offsets, &mtr); - ut_ad(trx == 0 || trx_is_referenced(trx)); + ut_ad(trx == 0 || trx->is_referenced()); } mtr_commit(&mtr); From 578ffcc5efad3da20a21056067755e0ae9e39267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 10:56:13 +0200 Subject: [PATCH 30/82] Skip mariabackup.huge_lsn if encryption is not available --- mysql-test/suite/mariabackup/huge_lsn.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 37be33916e7..baf577769c0 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -1,4 +1,5 @@ --source include/not_embedded.inc +--source include/have_file_key_management.inc --echo # --echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" From 30ecd2884a1464a0848dde1d34af84d194d1676a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 12:12:31 +0200 Subject: [PATCH 31/82] Fix compilation warnings for libmariadb --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index 8231a0add6a..f3944bbd36a 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit 8231a0add6a9c2e12a758db2e7072d42deef354b +Subproject commit f3944bbd36af729b2f13313587018679c57de67d From 773c3ceb573ea5ae8506237d3648412251ce02e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 15:39:36 +0200 Subject: [PATCH 32/82] MDEV-14824 Assertion `!trx_is_started(trx)' failed in innobase_start_trx_and_assign_read_view In CREATE SEQUENCE or CREATE TEMPORARY SEQUENCE, we should not start an InnoDB transaction for inserting the sequence status record into the underlying no-rollback table. Because we did this, a debug assertion failure would fail in START TRANSACTION WITH CONSISTENT SNAPSHOT after CREATE TEMPORARY SEQUENCE was executed. row_ins_step(): Do not start the transaction. Let the caller do that. que_thr_step(): Start the transaction before calling row_ins_step(). row_ins_clust_index_entry(): Skip locking and undo logging for no-rollback tables, even for temporary no-rollback tables. row_ins_index_entry(): Allow trx->id==0 for no-rollback tables. row_insert_for_mysql(): Do not start a transaction for no-rollback tables. --- mysql-test/suite/sql_sequence/create.result | 1 + mysql-test/suite/sql_sequence/create.test | 1 + storage/innobase/que/que0que.cc | 3 ++- storage/innobase/row/row0ins.cc | 12 +++++------- storage/innobase/row/row0mysql.cc | 6 ++++-- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/mysql-test/suite/sql_sequence/create.result b/mysql-test/suite/sql_sequence/create.result index 015b5ac762b..20d16822575 100644 --- a/mysql-test/suite/sql_sequence/create.result +++ b/mysql-test/suite/sql_sequence/create.result @@ -482,6 +482,7 @@ select previous value for t1; previous value for t1 1 CREATE TEMPORARY SEQUENCE t1 start with 100 minvalue 100 maxvalue 200 increment by 1 cache 10 engine=innodb; +START TRANSACTION WITH CONSISTENT SNAPSHOT; select previous value for t1; previous value for t1 NULL diff --git a/mysql-test/suite/sql_sequence/create.test b/mysql-test/suite/sql_sequence/create.test index 6696e78db92..91411bebfde 100644 --- a/mysql-test/suite/sql_sequence/create.test +++ b/mysql-test/suite/sql_sequence/create.test @@ -362,6 +362,7 @@ CREATE SEQUENCE t1 start with 1 minvalue 1 maxvalue 10 increment by 1 cache 10 e select next value for t1; select previous value for t1; CREATE TEMPORARY SEQUENCE t1 start with 100 minvalue 100 maxvalue 200 increment by 1 cache 10 engine=innodb; +START TRANSACTION WITH CONSISTENT SNAPSHOT; select previous value for t1; select next value for t1; select previous value for t1; diff --git a/storage/innobase/que/que0que.cc b/storage/innobase/que/que0que.cc index 5a3af9dfaeb..a45ac188f2f 100644 --- a/storage/innobase/que/que0que.cc +++ b/storage/innobase/que/que0que.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1030,6 +1030,7 @@ que_thr_step( } else if (type == QUE_NODE_SELECT) { thr = row_sel_step(thr); } else if (type == QUE_NODE_INSERT) { + trx_start_if_not_started_xa(thr_get_trx(thr), true); thr = row_ins_step(thr); } else if (type == QUE_NODE_UPDATE) { trx_start_if_not_started_xa(thr_get_trx(thr), true); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index c111aa8e776..a62b888a8b4 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2017, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3193,9 +3193,9 @@ row_ins_clust_index_entry( /* Try first optimistic descent to the B-tree */ log_free_check(); - const ulint flags = dict_table_is_temporary(index->table) - ? BTR_NO_LOCKING_FLAG - : index->table->no_rollback() ? BTR_NO_ROLLBACK : 0; + const ulint flags = index->table->no_rollback() ? BTR_NO_ROLLBACK + : dict_table_is_temporary(index->table) + ? BTR_NO_LOCKING_FLAG : 0; err = row_ins_clust_index_entry_low( flags, BTR_MODIFY_LEAF, index, n_uniq, entry, @@ -3304,7 +3304,7 @@ row_ins_index_entry( dtuple_t* entry, /*!< in/out: index entry to insert */ que_thr_t* thr) /*!< in: query thread */ { - ut_ad(thr_get_trx(thr)->id != 0); + ut_ad(thr_get_trx(thr)->id || index->table->no_rollback()); DBUG_EXECUTE_IF("row_ins_index_entry_timeout", { DBUG_SET("-d,row_ins_index_entry_timeout"); @@ -3764,8 +3764,6 @@ row_ins_step( trx = thr_get_trx(thr); - trx_start_if_not_started_xa(trx, true); - node = static_cast(thr->run_node); ut_ad(que_node_get_type(node) == QUE_NODE_INSERT); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 4b06c81ff6c..99f4d37dc6c 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1453,7 +1453,9 @@ row_insert_for_mysql( row_mysql_delay_if_needed(); - trx_start_if_not_started_xa(trx, true); + if (!table->no_rollback()) { + trx_start_if_not_started_xa(trx, true); + } row_get_prebuilt_insert_row(prebuilt); node = prebuilt->ins_node; From bf7719111fff5167e160abf869c03a81a3a39592 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 11 Jan 2018 17:09:51 +0200 Subject: [PATCH 33/82] Removed duplicated copyright message --- BUILD/compile-pentium64-asan-max | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/BUILD/compile-pentium64-asan-max b/BUILD/compile-pentium64-asan-max index db595b418df..cd5a292d17c 100755 --- a/BUILD/compile-pentium64-asan-max +++ b/BUILD/compile-pentium64-asan-max @@ -1,20 +1,5 @@ #! /bin/sh -# Copyright (c) 2005, 2010, Oracle and/or its affiliates. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -# Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2018, MariaDB Corporation. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 21239bb0fd2859968d3c42dcc56712a8978b6207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 22:54:22 +0200 Subject: [PATCH 34/82] After-merge fix to innodb.log_corruption --- .../encryption/r/innodb_encrypt_log_corruption.result | 4 ++-- mysql-test/suite/innodb/t/log_corruption.test | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result index e85ad3c9db7..23e8a1c98c8 100644 --- a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result +++ b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result @@ -124,8 +124,8 @@ SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); COUNT(*) -0 -FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err +1 +FOUND 2 /InnoDB: Upgrading redo log:/ in mysqld.1.err # Minimal MariaDB 10.1.21 encrypted redo log SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test index a1d85b9f698..ca5d945afb7 100644 --- a/mysql-test/suite/innodb/t/log_corruption.test +++ b/mysql-test/suite/innodb/t/log_corruption.test @@ -428,8 +428,15 @@ print OUT pack("Nx[5]nx[5]", 1, 0x1286); print OUT "ibbackup was here!!!1!"; print OUT pack("x[470]N", 0x52b54540); # In encryption.innodb_log_corruption the previous step would -# replace the block with an encrypted one. Rewrite it as it was. -die unless seek(OUT, 0x800, 0); +# replace the block with an encrypted one and update the checkpoint. +# Restore them. +# invalid (all-zero) checkpoint page 1 and an empty log page +print OUT chr(0) x 1024; +# valid checkpoint block 2 +print OUT pack("x[12]NNNx[264]", 0x12860c, 0, 0x80c); +# pointer to the MLOG_CHECKPOINT record, and checkpoint page checksum +print OUT pack("H*x[204]NNN", "590DBAACFE922582", 0x128612, 0, 0x101741b); +# log page print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 1, 0x46c8a2a2); close OUT or die; EOF From 5fce14dad02be5b23a8718a9a94fb7de00a7bb1e Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 11 Jan 2018 23:55:13 +0200 Subject: [PATCH 35/82] Removed wrong DBUG_DUMP that accessed not initialized memory. --- sql/field.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/field.cc b/sql/field.cc index e723b62163a..7b8ec18cad6 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -8294,6 +8294,7 @@ uchar *Field_blob::pack(uchar *to, const uchar *from, uint max_length) @return New pointer into memory based on from + length of the data */ + const uchar *Field_blob::unpack(uchar *to, const uchar *from, const uchar *from_end, uint param_data) { @@ -8311,7 +8312,6 @@ const uchar *Field_blob::unpack(uchar *to, const uchar *from, DBUG_RETURN(0); store(reinterpret_cast(from) + master_packlength, length, field_charset); - DBUG_DUMP("record", to, table->s->reclength); DBUG_RETURN(from + master_packlength + length); } From 3dc3ab1a3048484910ca8acccaf76c71b080e533 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Thu, 11 Jan 2018 23:56:54 +0200 Subject: [PATCH 36/82] Added checking that row events ends with a proper end block Problems -------- The slave io thread did not conduct integrity check for a group of row-based events. Specifically it tolerates missed terminal block event that must be flagged with STMT_END. Failure to react on its loss can confuse the applier thread in various ways. Another potential issue was that there were no check of impossible second in row Gtid-log-event while the slave io thread is receiving to be skipped events after reconnect. Fixes ----- The slave io thread is made by this patch to track the rows event STMT_END status. Whenever at next event reading the IO thread finds out that a preceding Rows event did not actually had the flag, an explicit error is issued. Replication can be resumed after the source of failure is eliminated, see a provided test. Note that currently the row-based group integrity check excludes the compressed version 2 Rows events (which are not generated by MariaDB master). Its uncompressed counterpart is manually tested. The 2nd issue is covered to produce an error in case the io thread receives a successive Gtid_log_event while it is post-reconnect skipping. --- .../r/rpl_row_end_of_statement_loss.result | 42 ++++ .../rpl_row_end_of_statement_loss-master.opt | 2 + .../rpl/t/rpl_row_end_of_statement_loss.test | 66 ++++++ sql/rpl_mi.h | 21 ++ sql/slave.cc | 202 +++++++++++++++++- 5 files changed, 322 insertions(+), 11 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result create mode 100644 mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt create mode 100644 mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test diff --git a/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result b/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result new file mode 100644 index 00000000000..dc6a67b48d2 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result @@ -0,0 +1,42 @@ +include/master-slave.inc +[connection master] +connection slave; +call mtr.add_suppression("Slave IO thread did not receive an expected Rows-log end-of-statement"); +call mtr.add_suppression("Relay log write failure: could not queue event from master"); +SET @save_debug= @@global.debug; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +include/stop_slave.inc +CHANGE MASTER TO master_host = '127.0.0.1', master_port = MASTER_PORT, MASTER_USE_GTID=SLAVE_POS; +connection master; +CREATE TABLE t (a INT, b text(8192));; +INSERT INTO t values (1, repeat('b', 8192)), (1, repeat('b', 8192)); +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1595] +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +include/start_slave.inc +connection master; +connection slave; +connection slave; +include/stop_slave.inc +connection master; +SET @save_log_bin_compress= @@GLOBAL.log_bin_compress; +SET @save_log_bin_compress_min_len= @@GLOBAL.log_bin_compress_min_len; +SET @@GLOBAL.log_bin_compress=ON; +SET @@GLOBAL.log_bin_compress_min_len=10; +INSERT INTO t values (2, repeat('b', 8192)), (2, repeat('b', 8192)); +connection slave; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1595] +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +include/start_slave.inc +connection master; +connection slave; +connection master; +SET @@GLOBAL.log_bin_compress= @save_log_bin_compress; +SET @@GLOBAL.log_bin_compress_min_len= @save_log_bin_compress_min_len; +DROP TABLE t; +connection slave; +SET GLOBAL debug_dbug= @save_debug; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt new file mode 100644 index 00000000000..144bbca0730 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt @@ -0,0 +1,2 @@ +--binlog-row-event-max-size=8192 + diff --git a/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test new file mode 100644 index 00000000000..5b2d99f3bf1 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test @@ -0,0 +1,66 @@ +--source include/have_debug.inc +--source include/have_binlog_format_row.inc +--source include/master-slave.inc + +# Loss of STMT_END flagged event must error out the IO thread +--connection slave +call mtr.add_suppression("Slave IO thread did not receive an expected Rows-log end-of-statement"); +call mtr.add_suppression("Relay log write failure: could not queue event from master"); + +SET @save_debug= @@global.debug; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +--source include/stop_slave.inc +--replace_result $MASTER_MYPORT MASTER_PORT +--eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $MASTER_MYPORT, MASTER_USE_GTID=SLAVE_POS + +--connection master +--let $max_row_size=8192 +--eval CREATE TABLE t (a INT, b text($max_row_size)); +--eval INSERT INTO t values (1, repeat('b', $max_row_size)), (1, repeat('b', $max_row_size)) + +# Prove that the missed STMT_END marked rows-event causes the io thread stop. +--connection slave +START SLAVE IO_THREAD; +--let $slave_io_errno=1595 +--source include/wait_for_slave_io_error.inc +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +--source include/start_slave.inc + +--connection master +sync_slave_with_master; + +# Compressed version of the above +--connection slave +--source include/stop_slave.inc + +--connection master +SET @save_log_bin_compress= @@GLOBAL.log_bin_compress; +SET @save_log_bin_compress_min_len= @@GLOBAL.log_bin_compress_min_len; + +SET @@GLOBAL.log_bin_compress=ON; +SET @@GLOBAL.log_bin_compress_min_len=10; + +--eval INSERT INTO t values (2, repeat('b', $max_row_size)), (2, repeat('b', $max_row_size)) + +# Prove that the missed STMT_END marked rows-event causes the io thread stop. +--connection slave +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +START SLAVE IO_THREAD; +--let $slave_io_errno=1595 +--source include/wait_for_slave_io_error.inc +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +--source include/start_slave.inc + +--connection master +sync_slave_with_master; + +# cleanup + +--connection master +SET @@GLOBAL.log_bin_compress= @save_log_bin_compress; +SET @@GLOBAL.log_bin_compress_min_len= @save_log_bin_compress_min_len; +DROP TABLE t; +sync_slave_with_master; +SET GLOBAL debug_dbug= @save_debug; + +--source include/rpl_end.inc diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index ccc1be6e5ce..b304e45f86a 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -133,6 +133,19 @@ public: extern TYPELIB slave_parallel_mode_typelib; +typedef struct st_rows_event_tracker +{ + char binlog_file_name[FN_REFLEN]; + my_off_t first_seen; + my_off_t last_seen; + bool stmt_end_seen; + void update(const char* file_name, size_t pos, + const char* buf, + const Format_description_log_event *fdle); + void reset(); + bool check_and_report(const char* file_name, size_t pos); +} Rows_event_tracker; + /***************************************************************************** Replication IO Thread @@ -301,6 +314,14 @@ class Master_info : public Slave_reporting_capability uint64 gtid_reconnect_event_skip_count; /* gtid_event_seen is false until we receive first GTID event from master. */ bool gtid_event_seen; + /** + The struct holds some history of Rows- log-event reading/queuing + by the receiver thread. Its fields are updated per each such event + at time of queue_event(), and they are checked to detect + the Rows- event group integrity violation at time of first non-Rows- + event gets handled. + */ + Rows_event_tracker rows_event_tracker; bool in_start_all_slaves, in_stop_all_slaves; bool in_flush_all_relay_logs; uint users; /* Active user for object */ diff --git a/sql/slave.cc b/sql/slave.cc index c686553fdd5..e5c502c2de5 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3349,7 +3349,8 @@ static ulong read_event(MYSQL* mysql, Master_info *mi, bool* suppress_warnings, we suppress prints to .err file as long as the reconnect happens without problems */ - *suppress_warnings= TRUE; + *suppress_warnings= + global_system_variables.log_warnings < 2 ? TRUE : FALSE; } else { @@ -4274,6 +4275,7 @@ pthread_handler_t handle_slave_io(void *arg) mi->abort_slave = 0; mysql_mutex_unlock(&mi->run_lock); mysql_cond_broadcast(&mi->start_cond); + mi->rows_event_tracker.reset(); DBUG_PRINT("master_info",("log_file_name: '%s' position: %llu", mi->master_log_name, mi->master_log_pos)); @@ -4356,6 +4358,10 @@ connected: */ mi->gtid_reconnect_event_skip_count= mi->events_queued_since_last_gtid; mi->gtid_event_seen= false; + /* + Reset stale state of the rows-event group tracker at reconnect. + */ + mi->rows_event_tracker.reset(); } #ifdef ENABLED_DEBUG_SYNC @@ -5752,7 +5758,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) char* new_buf = NULL; char new_buf_arr[4096]; bool is_malloc = false; - + bool is_rows_event= false; /* FD_q must have been prepared for the first R_a event inside get_master_version_and_clock() @@ -6186,11 +6192,11 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) got_gtid_event= true; if (mi->using_gtid == Master_info::USE_GTID_NO) goto default_action; - if (unlikely(!mi->gtid_event_seen)) + if (unlikely(mi->gtid_reconnect_event_skip_count)) { - mi->gtid_event_seen= true; - if (mi->gtid_reconnect_event_skip_count) + if (likely(!mi->gtid_event_seen)) { + mi->gtid_event_seen= true; /* If we are reconnecting, and we need to skip a partial event group already queued to the relay log before the reconnect, then we check @@ -6219,13 +6225,45 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) rpl_slave_state_tostring_helper(&error_msg, &event_gtid, &first); goto err; } + if (global_system_variables.log_warnings > 1) + { + bool first= true; + StringBuffer<1024> gtid_text; + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + sql_print_information("Slave IO thread is reconnected to " + "receive Gtid_log_event %s. It is to skip %llu " + "already received events including the gtid one", + gtid_text.ptr(), + mi->events_queued_since_last_gtid); + } + goto default_action; + } + else + { + bool first; + StringBuffer<1024> gtid_text; + + gtid_text.append(STRING_WITH_LEN("Last received gtid: ")); + first= true; + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + gtid_text.append(STRING_WITH_LEN(", currently received: ")); + first= true; + rpl_slave_state_tostring_helper(>id_text, &event_gtid, &first); + + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + sql_print_error("Slave IO thread has received a new Gtid_log_event " + "while skipping already logged events " + "after reconnect. %s. %llu remains to be skipped. " + "The number of originally read events was %llu", + gtid_text.ptr(), + mi->gtid_reconnect_event_skip_count, + mi->events_queued_since_last_gtid); + goto err; } } - - if (unlikely(mi->gtid_reconnect_event_skip_count)) - { - goto default_action; - } + mi->gtid_event_seen= true; /* We have successfully queued to relay log everything before this GTID, so @@ -6292,8 +6330,34 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) goto err; } } - buf = new_buf; is_compress_event = true; + buf = new_buf; + /* + As we are uncertain about compressed V2 rows events, we don't track + them + */ + if (LOG_EVENT_IS_ROW_V2((Log_event_type) buf[EVENT_TYPE_OFFSET])) + goto default_action; + /* fall through */ + case WRITE_ROWS_EVENT_V1: + case UPDATE_ROWS_EVENT_V1: + case DELETE_ROWS_EVENT_V1: + case WRITE_ROWS_EVENT: + case UPDATE_ROWS_EVENT: + case DELETE_ROWS_EVENT: + { + is_rows_event= true; + mi->rows_event_tracker.update(mi->master_log_name, + mi->master_log_pos, + buf, + mi->rli.relay_log. + description_event_for_queue); + + DBUG_EXECUTE_IF("simulate_stmt_end_rows_event_loss", + { + mi->rows_event_tracker.stmt_end_seen= false; + }); + } goto default_action; #ifndef DBUG_OFF @@ -6351,6 +6415,21 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) break; } + /* + Integrity of Rows- event group check. + A sequence of Rows- events must end with STMT_END_F flagged one. + Even when Heartbeat event interrupts Rows- events flow this must indicate a + malfunction e.g logging on the master. + */ + if (((uchar) buf[EVENT_TYPE_OFFSET] != HEARTBEAT_LOG_EVENT) && + !is_rows_event && + mi->rows_event_tracker.check_and_report(mi->master_log_name, + mi->master_log_pos)) + { + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + goto err; + } + /* If we filter events master-side (eg. @@skip_replication), we will see holes in the event positions from the master. If we see such a hole, adjust @@ -6519,6 +6598,21 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) The whole of the current event group is queued. So in case of reconnect we can start from after the current GTID. */ + if (mi->gtid_reconnect_event_skip_count) + { + bool first= true; + StringBuffer<1024> gtid_text; + + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + sql_print_error("Slave IO thread received a terminal event from " + "group %s whose retrieval was interrupted " + "with reconnect. We still had %llu events to read. " + "The number of originally read events was %llu", + gtid_text.ptr(), + mi->gtid_reconnect_event_skip_count, + mi->events_queued_since_last_gtid); + } mi->gtid_current_pos.update(&mi->last_queued_gtid); mi->events_queued_since_last_gtid= 0; @@ -7518,6 +7612,92 @@ bool rpl_master_erroneous_autoinc(THD *thd) return FALSE; } + +static bool get_row_event_stmt_end(const char* buf, + const Format_description_log_event *fdle) +{ + uint8 const common_header_len= fdle->common_header_len; + Log_event_type event_type= (Log_event_type)(uchar)buf[EVENT_TYPE_OFFSET]; + + uint8 const post_header_len= fdle->post_header_len[event_type-1]; + const char *flag_start= buf + common_header_len; + /* + The term 4 below signifies that master is of 'an intermediate source', see + Rows_log_event::Rows_log_event. + */ + flag_start += RW_MAPID_OFFSET + (post_header_len == 6) ? 4 : RW_FLAGS_OFFSET; + + return (uint2korr(flag_start) & Rows_log_event::STMT_END_F) != 0; +} + + +/* + Reset log event tracking data. +*/ + +void Rows_event_tracker::reset() +{ + binlog_file_name[0]= 0; + first_seen= last_seen= 0; + stmt_end_seen= false; +} + + +/* + Update log event tracking data. + + The first- and last- seen event binlog position get memorized, as + well as the end-of-statement status of the last one. +*/ + +void Rows_event_tracker::update(const char* file_name, size_t pos, + const char* buf, + const Format_description_log_event *fdle) +{ + if (!first_seen) + { + first_seen= pos; + strmake(binlog_file_name, file_name, sizeof(binlog_file_name) - 1); + } + last_seen= pos; + DBUG_ASSERT(stmt_end_seen == 0); // We can only have one + stmt_end_seen= get_row_event_stmt_end(buf, fdle); +}; + + +/** + The function is called at next event reading + after a sequence of Rows- log-events. It checks the end-of-statement status + of the past sequence to report on any isssue. + In the positive case the tracker gets reset. + + @return true when the Rows- event group integrity found compromised, + false otherwise. +*/ +bool Rows_event_tracker::check_and_report(const char* file_name, + size_t pos) +{ + if (last_seen) + { + // there was at least one "block" event previously + if (!stmt_end_seen) + { + sql_print_error("Slave IO thread did not receive an expected " + "Rows-log end-of-statement for event starting " + "at log '%s' position %llu " + "whose last block was seen at log '%s' position %llu. " + "The end-of-statement should have been delivered " + "before the current one at log '%s' position %llu", + binlog_file_name, first_seen, + binlog_file_name, last_seen, file_name, pos); + return true; + } + reset(); + } + + return false; +} + /** @} (end of group Replication) */ From d32f5be307d5593bf36d243de275af88ff8ee92a Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 12 Jan 2018 15:11:56 +0300 Subject: [PATCH 37/82] MDEV-14372: Fix and enable rocksdb.information_schema test - Make Rdb_binlog_manager::unpack_value to not have a stack overrun when it is reading invalid data (which it currently does as we in MariaDB do not store binlog coordinates under BINLOG_INFO_INDEX_NUMBER, see comments in MDEV-14892 for details). - We may need to store these coordinates in the future, so instead of removing the call of this function, let's make it work properly for all possible inputs. --- .../rocksdb/r/information_schema.result | 30 +++++++++---------- .../rocksdb/mysql-test/rocksdb/t/disabled.def | 1 - .../rocksdb/t/information_schema-master.opt | 2 +- .../rocksdb/t/information_schema.test | 12 ++++++-- storage/rocksdb/rdb_datadic.cc | 27 ++++++++++++++++- storage/rocksdb/rdb_datadic.h | 5 +++- storage/rocksdb/rdb_i_s.cc | 1 - 7 files changed, 56 insertions(+), 22 deletions(-) diff --git a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index 291effa832c..717fb0b970c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@ -1,6 +1,20 @@ DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; +create table t1 (a int) engine=rocksdb; +drop table t1; +select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; +TYPE NAME VALUE +MAX_INDEX_ID MAX_INDEX_ID max_index_id +CF_FLAGS 0 default [0] +CF_FLAGS 1 __system__ [0] +DDL_DROP_INDEX_ONGOING cf_id:0,index_id:max_index_id +select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; +count(*) +4 +select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; +CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; +INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE MAX_INDEX_ID MAX_INDEX_ID max_index_id @@ -9,20 +23,6 @@ CF_FLAGS 1 __system__ [0] select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; count(*) 3 -select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; -CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; -INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); -select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; -TYPE NAME VALUE -BINLOG FILE master-bin.000001 -BINLOG POS 1066 -BINLOG GTID uuid:5 -MAX_INDEX_ID MAX_INDEX_ID max_index_id -CF_FLAGS 0 default [0] -CF_FLAGS 1 __system__ [0] -select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; -count(*) -6 set global rocksdb_force_flush_memtable_now = true; set global rocksdb_compact_cf='default'; select case when VALUE-@keysIn >= 3 then 'true' else 'false' end from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; @@ -66,7 +66,7 @@ SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK'; Variable_name Value rocksdb_pause_background_work ON DROP TABLE t3; -cf_id:0,index_id:267 +cf_id:0,index_id:264 SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK'; Variable_name Value diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index dc2e04b93d7..359d1d3e082 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -72,7 +72,6 @@ blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api tes ## Tests that fail for some other reason ## -information_schema : MariaRocks: requires GTIDs mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs rpl_row_triggers : MariaRocks: requires GTIDs diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt index 40b14167e17..86379847638 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt @@ -1 +1 @@ ---binlog_format=row --gtid_mode=ON --enforce_gtid_consistency --log_slave_updates +--binlog_format=row --log_slave_updates diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index c8d9b538529..8746e589046 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@ -9,6 +9,13 @@ DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; --enable_warnings +# MariaDB: the following is for handling the case where the tests +# is started on a totally empty datadir, where no MyRocks table has +# ever been created). In that case, there is no MAX_INDEX_ID. +# Create/drop a table so that we do have MAX_INDEX_ID. +create table t1 (a int) engine=rocksdb; +drop table t1; + --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) --replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; @@ -19,9 +26,10 @@ select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); ---let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +# No binlog coordinates in MariaDB: --let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) ---replace_result $MASTER_UUID uuid $max_index_id max_index_id +# No binlog coordinates in MariaDB: --replace_result $MASTER_UUID uuid $max_index_id max_index_id +--replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index f6f1406e80c..b2f5af705a3 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -4261,7 +4261,7 @@ bool Rdb_binlog_manager::read(char *const binlog_name, std::string value; rocksdb::Status status = m_dict->get_value(m_key_slice, &value); if (status.ok()) { - if (!unpack_value((const uchar *)value.c_str(), binlog_name, binlog_pos, + if (!unpack_value((const uchar *)value.c_str(), value.size(), binlog_name, binlog_pos, binlog_gtid)) ret = true; } @@ -4331,35 +4331,60 @@ Rdb_binlog_manager::pack_value(uchar *const buf, const char *const binlog_name, @return true on error */ bool Rdb_binlog_manager::unpack_value(const uchar *const value, + size_t value_size_arg, char *const binlog_name, my_off_t *const binlog_pos, char *const binlog_gtid) const { uint pack_len = 0; + intmax_t value_size= value_size_arg; DBUG_ASSERT(binlog_pos != nullptr); + if ((value_size -= Rdb_key_def::VERSION_SIZE) < 0) + return true; // read version const uint16_t version = rdb_netbuf_to_uint16(value); + pack_len += Rdb_key_def::VERSION_SIZE; if (version != Rdb_key_def::BINLOG_INFO_INDEX_NUMBER_VERSION) return true; + if ((value_size -= sizeof(uint16)) < 0) + return true; + // read binlog file name length const uint16_t binlog_name_len = rdb_netbuf_to_uint16(value + pack_len); pack_len += sizeof(uint16); + + if (binlog_name_len >= (FN_REFLEN+1)) + return true; + + if ((value_size -= binlog_name_len) < 0) + return true; + if (binlog_name_len) { // read and set binlog name memcpy(binlog_name, value + pack_len, binlog_name_len); binlog_name[binlog_name_len] = '\0'; pack_len += binlog_name_len; + if ((value_size -= sizeof(uint32)) < 0) + return true; // read and set binlog pos *binlog_pos = rdb_netbuf_to_uint32(value + pack_len); pack_len += sizeof(uint32); + if ((value_size -= sizeof(uint16)) < 0) + return true; // read gtid length const uint16_t binlog_gtid_len = rdb_netbuf_to_uint16(value + pack_len); pack_len += sizeof(uint16); + + if (binlog_gtid_len >= GTID_BUF_LEN) + return true; + if ((value_size -= binlog_gtid_len) < 0) + return true; + if (binlog_gtid && binlog_gtid_len > 0) { // read and set gtid memcpy(binlog_gtid, value + pack_len, binlog_gtid_len); diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index 5796132de39..223f61edb43 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -46,6 +46,8 @@ class Rdb_field_packing; class Rdb_cf_manager; class Rdb_ddl_manager; +const uint32_t GTID_BUF_LEN = 60; + /* @brief Field packing context. @@ -1154,7 +1156,8 @@ private: rocksdb::Slice pack_value(uchar *const buf, const char *const binlog_name, const my_off_t &binlog_pos, const char *const binlog_gtid) const; - bool unpack_value(const uchar *const value, char *const binlog_name, + bool unpack_value(const uchar *const value, size_t value_size, + char *const binlog_name, my_off_t *const binlog_pos, char *const binlog_gtid) const; std::atomic m_slave_gtid_info_tbl; diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index 424a9e6c1f4..f8ddcb00fb3 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -729,7 +729,6 @@ static int rdb_i_s_global_info_fill_table( DBUG_ASSERT(tables != nullptr); static const uint32_t INT_BUF_LEN = 21; - static const uint32_t GTID_BUF_LEN = 60; static const uint32_t CF_ID_INDEX_BUF_LEN = 60; int ret = 0; From c481fc9ca788a3ac1d61ed6336208c0e2c1cde78 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 12 Jan 2018 15:57:08 +0300 Subject: [PATCH 38/82] Change MyRocks maturity from Alpha to Beta --- storage/rocksdb/ha_rocksdb.h | 2 +- .../rocksdb/r/mariadb_port_fixes.result | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 82819bbf7b2..4432a4de8d1 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -1412,6 +1412,6 @@ private: Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &); }; -const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_ALPHA; +const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_BETA; } // namespace myrocks diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index 9952314cd2c..021373be02a 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -69,15 +69,15 @@ set global rocksdb_strict_collation_check=@tmp_rscc; # select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; plugin_name plugin_maturity -ROCKSDB Alpha -ROCKSDB_CFSTATS Alpha -ROCKSDB_DBSTATS Alpha -ROCKSDB_PERF_CONTEXT Alpha -ROCKSDB_PERF_CONTEXT_GLOBAL Alpha -ROCKSDB_CF_OPTIONS Alpha -ROCKSDB_COMPACTION_STATS Alpha -ROCKSDB_GLOBAL_INFO Alpha -ROCKSDB_DDL Alpha -ROCKSDB_INDEX_FILE_MAP Alpha -ROCKSDB_LOCKS Alpha -ROCKSDB_TRX Alpha +ROCKSDB Beta +ROCKSDB_CFSTATS Beta +ROCKSDB_DBSTATS Beta +ROCKSDB_PERF_CONTEXT Beta +ROCKSDB_PERF_CONTEXT_GLOBAL Beta +ROCKSDB_CF_OPTIONS Beta +ROCKSDB_COMPACTION_STATS Beta +ROCKSDB_GLOBAL_INFO Beta +ROCKSDB_DDL Beta +ROCKSDB_INDEX_FILE_MAP Beta +ROCKSDB_LOCKS Beta +ROCKSDB_TRX Beta From 028e2ddc549fb7573b11392c87b9e8faf1cbf38b Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Fri, 12 Jan 2018 19:16:36 +0530 Subject: [PATCH 39/82] Added a missing result file to the rocksdb_sys_vars result suite --- .../rocksdb_sys_vars/include/rocksdb_sys_var.inc | 1 + .../rocksdb_sys_vars/r/rocksdb_git_hash_basic.result | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc b/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc index 6ba93026674..db0abc57358 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc @@ -10,6 +10,7 @@ --eval SET @start_global_value = @@global.$sys_var if (!$suppress_default_value) { + --replace_regex /[a-f0-9]{40}/#/ SELECT @start_global_value; if ($session) { diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result new file mode 100644 index 00000000000..bbcfa1417eb --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result @@ -0,0 +1,7 @@ +SET @start_global_value = @@global.ROCKSDB_GIT_HASH; +SELECT @start_global_value; +@start_global_value +# +"Trying to set variable @@global.ROCKSDB_GIT_HASH to 444. It should fail because it is readonly." +SET @@global.ROCKSDB_GIT_HASH = 444; +ERROR HY000: Variable 'rocksdb_git_hash' is a read only variable From c9c28bef3cb4cf21a6fee46f5ea3339867fbb07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Wed, 15 Nov 2017 12:37:32 +0800 Subject: [PATCH 40/82] Minor spelling fixes in code comments, docs and output This commit does not touch any variable names or any other actual code, and thus should not in any way affect how the code works. --- BUILD-CMAKE | 2 +- Docs/sp-imp-spec.txt | 4 ++-- cmake/cpack_rpm.cmake | 2 +- configure.cmake | 2 +- debian/autobake-deb.sh | 2 +- debian/mariadb-server-10.3.mysql.init | 2 +- include/m_ctype.h | 4 ++-- include/my_atomic.h | 2 +- include/my_global.h | 8 ++++---- include/my_pthread.h | 4 ++-- include/mysql/service_thd_rnd.h | 2 +- include/mysql_com.h | 4 ++-- man/mysql-test-run.pl.1 | 4 ++-- man/mysql.1 | 2 +- mysys/my_atomic_writes.c | 2 +- mysys/my_context.c | 2 +- mysys/my_getopt.c | 2 +- mysys/thr_mutex.c | 4 ++-- strings/ctype-uca.c | 6 +++--- strings/decimal.c | 2 +- win/packaging/ca/CustomAction.cpp | 2 +- 21 files changed, 32 insertions(+), 32 deletions(-) diff --git a/BUILD-CMAKE b/BUILD-CMAKE index c95482cf619..aedce4400c6 100644 --- a/BUILD-CMAKE +++ b/BUILD-CMAKE @@ -97,7 +97,7 @@ shell>cd ../build shell>cmake ../src Note: if a directory was used for in-source build, out-of-source will -not work. To reenable out-of-source build, remove /CMakeCache.txt +not work. To re-enable out-of-source build, remove /CMakeCache.txt file. diff --git a/Docs/sp-imp-spec.txt b/Docs/sp-imp-spec.txt index f2794e14793..0526cc43f81 100644 --- a/Docs/sp-imp-spec.txt +++ b/Docs/sp-imp-spec.txt @@ -323,7 +323,7 @@ One "obvious" solution would be to simply push "mysql.proc" to the list of tables used by the query, but this implies a "join" with this table if the query is a select, so it doesn't work (and we can't exclude this - table easily; since a priviledged used might in fact want to search + table easily; since a privileged used might in fact want to search the proc table). Another solution would of course be to allow the opening and closing of the mysql.proc table during a query execution, but this it not @@ -400,7 +400,7 @@ instruction. Calling and returning from a CONTINUE handler poses some special - problems. Since we need to return to the point after its invokation, + problems. Since we need to return to the point after its invocation, we push the return location on a stack in the sp_rcontext (this is done by the exectution loop). The handler then ends with a special instruction, sp_instr_hreturn, which returns to this location. diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 5b8aa36ff0d..513f3671b5d 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -201,7 +201,7 @@ ALTERNATIVE_NAME("server" "mysql-server") ALTERNATIVE_NAME("test" "mysql-test") # Argh! Different distributions call packages differently, to be a drop-in -# replacement we have to fake distribution-speficic dependencies +# replacement we have to fake distribution-specificic dependencies IF(RPM MATCHES "(rhel|centos)6") ALTERNATIVE_NAME("client" "mysql") diff --git a/configure.cmake b/configure.cmake index 7258077d68b..b62ac12e634 100644 --- a/configure.cmake +++ b/configure.cmake @@ -488,7 +488,7 @@ int main() { # -# Test for endianess +# Test for endianness # INCLUDE(TestBigEndian) IF(APPLE) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index cc5b04188f4..1a709eb8284 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -33,7 +33,7 @@ fi # Look up distro-version specific stuff # # Always keep the actual packaging as up-to-date as possible following the latest -# Debian policy and targetting Debian Sid. Then case-by-case run in autobake-deb.sh +# Debian policy and targeting Debian Sid. Then case-by-case run in autobake-deb.sh # tests for backwards compatibility and strip away parts on older builders. # If iproute2 is not available (before Debian Jessie and Ubuntu Trusty) diff --git a/debian/mariadb-server-10.3.mysql.init b/debian/mariadb-server-10.3.mysql.init index d7f6d58e8ca..35a52d5d8db 100644 --- a/debian/mariadb-server-10.3.mysql.init +++ b/debian/mariadb-server-10.3.mysql.init @@ -25,7 +25,7 @@ SELF=$(cd $(dirname $0); pwd -P)/$(basename $0) MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf" -# priority can be overriden and "-s" adds output to stderr +# priority can be overridden and "-s" adds output to stderr ERR_LOGGER="logger -p daemon.err -t /etc/init.d/mysql -i" if [ -f /etc/default/mysql ]; then diff --git a/include/m_ctype.h b/include/m_ctype.h index b78443f88f8..da52b4c15a0 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -376,7 +376,7 @@ typedef size_t (*my_charset_conv_case)(CHARSET_INFO *, A structure to return the statistics of a native string copying, when no Unicode conversion is involved. - The stucture is OK to be unitialized before calling a copying routine. + The stucture is OK to be uninitialized before calling a copying routine. A copying routine must populate the structure as follows: - m_source_end_pos must be set by to a non-NULL value in the range of the input string. @@ -425,7 +425,7 @@ struct my_charset_handler_st my_charset_conv_case caseup; my_charset_conv_case casedn; - /* Charset dependant snprintf() */ + /* Charset dependent snprintf() */ size_t (*snprintf)(CHARSET_INFO *, char *to, size_t n, const char *fmt, ...) ATTRIBUTE_FORMAT_FPTR(printf, 4, 5); diff --git a/include/my_atomic.h b/include/my_atomic.h index 896dc2b5c33..1f5d85fcf12 100644 --- a/include/my_atomic.h +++ b/include/my_atomic.h @@ -101,7 +101,7 @@ sequentially-consistent operation ordering. We choose implementation as follows: on Windows using Visual C++ the native - implementation should be preferrable. When using gcc we prefer the Solaris + implementation should be preferable. When using gcc we prefer the Solaris implementation before the gcc because of stability preference, we choose gcc builtins if available. */ diff --git a/include/my_global.h b/include/my_global.h index 7986a0e21dd..a96c947cf44 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -178,7 +178,7 @@ /* The macros below are borrowed from include/linux/compiler.h in the - Linux kernel. Use them to indicate the likelyhood of the truthfulness + Linux kernel. Use them to indicate the likelihood of the truthfulness of a condition. This serves two purposes - newer versions of gcc will be able to optimize for branch predication, which could yield siginficant performance gains in frequently executed sections of the code, and the @@ -527,7 +527,7 @@ typedef SOCKET my_socket; typedef int my_socket; /* File descriptor for sockets */ #define INVALID_SOCKET -1 #endif -/* Type for fuctions that handles signals */ +/* Type for functions that handles signals */ #define sig_handler RETSIGTYPE C_MODE_START #ifdef HAVE_SIGHANDLER_T @@ -857,7 +857,7 @@ static inline double log2(double x) /* Max size that must be added to a so that we know Size to make - adressable obj. + addressable obj. */ #if SIZEOF_CHARP == 4 typedef long my_ptrdiff_t; @@ -869,7 +869,7 @@ typedef long long my_ptrdiff_t; #define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1)) #define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double)) #define ALIGN_MAX_UNIT (sizeof(double)) -/* Size to make adressable obj. */ +/* Size to make addressable obj. */ #define ALIGN_PTR(A, t) ((t*) MY_ALIGN((A), sizeof(double))) #define ADD_TO_PTR(ptr,size,type) (type) ((uchar*) (ptr)+size) #define PTR_BYTE_DIFF(A,B) (my_ptrdiff_t) ((uchar*) (A) - (uchar*) (B)) diff --git a/include/my_pthread.h b/include/my_pthread.h index 037ac0a062f..ecb201a60da 100644 --- a/include/my_pthread.h +++ b/include/my_pthread.h @@ -196,7 +196,7 @@ int sigwait(sigset_t *set, int *sig); #endif #if !defined(HAVE_SIGWAIT) && !defined(HAVE_rts_threads) && !defined(sigwait) && !defined(alpha_linux_port) && !defined(_AIX) -int sigwait(sigset_t *setp, int *sigp); /* Use our implemention */ +int sigwait(sigset_t *setp, int *sigp); /* Use our implementation */ #endif @@ -393,7 +393,7 @@ typedef struct st_safe_mutex_deadlock_t #ifdef SAFE_MUTEX_DETECT_DESTROY /* - Used to track the destroying of mutexes. This needs to be a seperate + Used to track the destroying of mutexes. This needs to be a separate structure because the safe_mutex_t structure could be freed before the mutexes are destroyed. */ diff --git a/include/mysql/service_thd_rnd.h b/include/mysql/service_thd_rnd.h index 21133c7889f..13fc1c16839 100644 --- a/include/mysql/service_thd_rnd.h +++ b/include/mysql/service_thd_rnd.h @@ -18,7 +18,7 @@ @file This service provides access to the thd-local random number generator. - It's preferrable over the global one, because concurrent threads + It's preferable over the global one, because concurrent threads can generate random numbers without fighting each other over the access to the shared rnd state. */ diff --git a/include/mysql_com.h b/include/mysql_com.h index 71858c93046..7f445502228 100644 --- a/include/mysql_com.h +++ b/include/mysql_com.h @@ -152,7 +152,7 @@ enum enum_indicator_type */ #define SCRAMBLE_LENGTH 20 #define SCRAMBLE_LENGTH_323 8 -/* length of password stored in the db: new passwords are preceeded with '*' */ +/* length of password stored in the db: new passwords are preceded with '*' */ #define SCRAMBLED_PASSWORD_CHAR_LENGTH (SCRAMBLE_LENGTH*2+1) #define SCRAMBLED_PASSWORD_CHAR_LENGTH_323 (SCRAMBLE_LENGTH_323*2) @@ -288,7 +288,7 @@ enum enum_indicator_type #endif /* - Gather all possible capabilites (flags) supported by the server + Gather all possible capabilities (flags) supported by the server MARIADB_* flags supported only by MariaDB connector(s). */ diff --git a/man/mysql-test-run.pl.1 b/man/mysql-test-run.pl.1 index 93fb42e251a..b928ec66587 100644 --- a/man/mysql-test-run.pl.1 +++ b/man/mysql-test-run.pl.1 @@ -276,7 +276,7 @@ T}:T{ Setting of a timeout in minutes or seconds, corresponding to command line option \fB\-\-\fR\fB\fIname\fR\fR\fB\-timeout\fR\&. - Avaliable timeout names are TESTCASE, + Available timeout names are TESTCASE, SUITE (both in minutes) and START, SHUTDOWN (both in seconds)\&. These variables are supported from @@ -1350,7 +1350,7 @@ for a description\&. .\" noreorder option: mysql-test-run.pl \fB\-\-noreorder\fR .sp -Do not reorder tests to reduce number of restarts, but run them in exactly the order given\&. If a whole suite is to be run, the tests are run in alphabetical order, though similiar combinations will be grouped together\&. If more than one suite is listed, the tests are run one suite at a time, in the order listed\&. +Do not reorder tests to reduce number of restarts, but run them in exactly the order given\&. If a whole suite is to be run, the tests are run in alphabetical order, though similar combinations will be grouped together\&. If more than one suite is listed, the tests are run one suite at a time, in the order listed\&. .RE .sp .RS 4 diff --git a/man/mysql.1 b/man/mysql.1 index b7f7a63bff4..f925fee454b 100644 --- a/man/mysql.1 +++ b/man/mysql.1 @@ -698,7 +698,7 @@ the section called \(lqMYSQL COMMANDS\(rq\&. Disabled by default\&. .\} .\" mysql: net-buffer-length option .\" net-buffer-length option: mysql -\fB\-\-net\-buffer\-lenght=\fR\fB\fIsize\fR\fR +\fB\-\-net\-buffer\-length=\fR\fB\fIsize\fR\fR .sp Set the buffer size for TCP/IP and socket communication\&. (Default value is 16KB\&.) .RE diff --git a/mysys/my_atomic_writes.c b/mysys/my_atomic_writes.c index 0b54a207713..7f1e353c121 100644 --- a/mysys/my_atomic_writes.c +++ b/mysys/my_atomic_writes.c @@ -259,7 +259,7 @@ static my_bool shannon_has_atomic_write(File file, int page_size) ************************************************************************/ /* - Initalize automic write sub systems. + Initialize automic write sub systems. Checks if we have any devices that supports atomic write */ diff --git a/mysys/my_context.c b/mysys/my_context.c index cf10738bdbd..5423f59d19b 100644 --- a/mysys/my_context.c +++ b/mysys/my_context.c @@ -707,7 +707,7 @@ my_context_continue(struct my_context *c) { /* This seems to be a common trick to run ConvertThreadToFiber() only on the - first occurence in a thread, in a way that works on multiple Windows + first occurrence in a thread, in a way that works on multiple Windows versions. */ void *current_fiber= GetCurrentFiber(); diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 0ea062988f6..9e617366ed5 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -236,7 +236,7 @@ int handle_options(int *argc, char ***argv, { is_cmdline_arg= 1; - /* save the separator too if skip unkown options */ + /* save the separator too if skip unknown options */ if (my_getopt_skip_unknown) (*argv)[argvpos++]= cur_arg; else diff --git a/mysys/thr_mutex.c b/mysys/thr_mutex.c index 49cb3ea600f..268a2b80f63 100644 --- a/mysys/thr_mutex.c +++ b/mysys/thr_mutex.c @@ -240,7 +240,7 @@ int safe_mutex_lock(safe_mutex_t *mp, myf my_flags, const char *file, if (!mp->file) { fprintf(stderr, - "safe_mutex: Trying to lock unitialized mutex at %s, line %d\n", + "safe_mutex: Trying to lock uninitialized mutex at %s, line %d\n", file, line); fflush(stderr); abort(); @@ -585,7 +585,7 @@ int safe_mutex_destroy(safe_mutex_t *mp, const char *file, uint line) if (!mp->file) { fprintf(stderr, - "safe_mutex: Trying to destroy unitialized mutex at %s, line %d\n", + "safe_mutex: Trying to destroy uninitialized mutex at %s, line %d\n", file, line); fflush(stderr); abort(); diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c index e3ed531df93..4386b6aa432 100644 --- a/strings/ctype-uca.c +++ b/strings/ctype-uca.c @@ -34034,7 +34034,7 @@ apply_one_rule(MY_CHARSET_LOADER *loader, /** Check if collation rules are valid, - i.e. characters are not outside of the collation suported range. + i.e. characters are not outside of the collation supported range. */ static int check_rules(MY_CHARSET_LOADER *loader, @@ -34170,7 +34170,7 @@ init_weight_level(MY_CHARSET_LOADER *loader, MY_COLL_RULES *rules, memcpy(dst->weights, src->weights, npages * sizeof(uint16 *)); /* - Calculate maximum lenghts for the pages which will be overwritten. + Calculate maximum lengths for the pages which will be overwritten. Mark pages that will be otherwriten as NULL. We'll allocate their own memory. */ @@ -34188,7 +34188,7 @@ init_weight_level(MY_CHARSET_LOADER *loader, MY_COLL_RULES *rules, { /* Not an expansion and not a contraction. - The page correspoding to r->curr[0] in "dst" + The page corresponding to r->curr[0] in "dst" will need at least the same amount of weights that r->base[0] has in "src". */ diff --git a/strings/decimal.c b/strings/decimal.c index 7db5111fc84..d7fd9a5722b 100644 --- a/strings/decimal.c +++ b/strings/decimal.c @@ -1694,7 +1694,7 @@ done: scale increment for '/' NOTE - returned valued may be larger than the actual buffer requred + returned valued may be larger than the actual buffer required in the operation, as decimal_result_size, by design, operates on precision/scale values only and not on the actual decimal number diff --git a/win/packaging/ca/CustomAction.cpp b/win/packaging/ca/CustomAction.cpp index c0062ddcdd1..9803b0b07a2 100644 --- a/win/packaging/ca/CustomAction.cpp +++ b/win/packaging/ca/CustomAction.cpp @@ -524,7 +524,7 @@ LExit: or 4GB(x64 OS). Fragmentation due to loaded modules, heap and stack - limit maximum size of continous memory block further, + limit maximum size of continuous memory block further, so that limit for 32 bit process is about 1200 on 32 bit OS or 2000 MB on 64 bit OS(found experimentally). */ From 2da191791260d5ca3adcb6857404912d52926ee2 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 12 Jan 2018 16:04:29 +0000 Subject: [PATCH 41/82] Attempt to eliminate race conditions in rocksdb.information_schema --- storage/rocksdb/mysql-test/rocksdb/r/information_schema.result | 2 ++ storage/rocksdb/mysql-test/rocksdb/t/information_schema.test | 2 ++ 2 files changed, 4 insertions(+) diff --git a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index 717fb0b970c..6850d8dff16 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=1; create table t1 (a int) engine=rocksdb; drop table t1; select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; @@ -12,6 +13,7 @@ DDL_DROP_INDEX_ONGOING cf_id:0,index_id:max_index_id select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; count(*) 4 +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index 8746e589046..2ffc186dd8f 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@ -13,6 +13,7 @@ DROP TABLE IF EXISTS t3; # is started on a totally empty datadir, where no MyRocks table has # ever been created). In that case, there is no MAX_INDEX_ID. # Create/drop a table so that we do have MAX_INDEX_ID. +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=1; create table t1 (a int) engine=rocksdb; drop table t1; @@ -20,6 +21,7 @@ drop table t1; --replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; From 3e6fcb6ac8e39593d441f3b60fbc21a88330f2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 12 Jan 2018 16:44:27 +0200 Subject: [PATCH 42/82] MDEV-14935 Remove bogus conditions related to not redo-logging PAGE_MAX_TRX_ID changes InnoDB originally skipped the redo logging of PAGE_MAX_TRX_ID changes until I enabled it in commit e76b873f24cd7cfc4b2d69b33103a6dd9601cbb8 that was part of MySQL 5.5.5 already. Later, when a more complete history of the InnoDB Plugin for MySQL 5.1 (aka branches/zip in the InnoDB subversion repository) and of the planned-to-be closed-source branches/innodb+ that became the basis of InnoDB in MySQL 5.5 was pushed to the MySQL source repository, the change was part of commit 509e761f06d6d7902bd5fdd0955447ed772af768: ------------------------------------------------------------------------ r5038 | marko | 2009-05-19 22:59:07 +0300 (Tue, 19 May 2009) | 30 lines branches/zip: Write PAGE_MAX_TRX_ID to the redo log. Otherwise, transactions that are started before the rollback of incomplete transactions has finished may have an inconsistent view of the secondary indexes. dict_index_is_sec_or_ibuf(): Auxiliary function for controlling updates and checks of PAGE_MAX_TRX_ID: check whether an index is a secondary index or the insert buffer tree. page_set_max_trx_id(), page_update_max_trx_id(), lock_rec_insert_check_and_lock(), lock_sec_rec_modify_check_and_lock(), btr_cur_ins_lock_and_undo(), btr_cur_upd_lock_and_undo(): Add the parameter mtr. page_set_max_trx_id(): Allow mtr to be NULL. When mtr==NULL, do not attempt to write to the redo log. This only occurs when creating a page or reorganizing a compressed page. In these cases, the PAGE_MAX_TRX_ID will be set correctly during the application of redo log records, even though there is no explicit log record about it. btr_discard_only_page_on_level(): Preserve PAGE_MAX_TRX_ID. This function should be unreachable, though. btr_cur_pessimistic_update(): Update PAGE_MAX_TRX_ID. Add some assertions for checking that PAGE_MAX_TRX_ID is set on all secondary index leaf pages. rb://115 tested by Michael, fixes Issue #211 ------------------------------------------------------------------------ After this fix, some bogus references to recv_recovery_is_on() remained. Also, some references could be replaced with references to index->is_dummy to prepare us for MDEV-14481 (background redo log apply). --- storage/innobase/lock/lock0lock.cc | 15 +++++---------- storage/innobase/page/page0cur.cc | 17 ++++++++++------- storage/innobase/row/row0upd.cc | 4 ++-- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 2a6167ec809..09b900d1554 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -459,11 +459,7 @@ lock_sec_rec_cons_read_sees( /* NOTE that we might call this function while holding the search system latch. */ - if (recv_recovery_is_on()) { - - return(false); - - } else if (dict_table_is_temporary(index->table)) { + if (dict_table_is_temporary(index->table)) { /* Temp-tables are not shared across connections and multiple transactions from different connections cannot simultaneously @@ -1518,7 +1514,7 @@ lock_sec_rec_some_has_impl( max trx id to the log, and therefore during recovery, this value for a page may be incorrect. */ - if (max_trx_id < trx_rw_min_trx_id() && !recv_recovery_is_on()) { + if (max_trx_id < trx_rw_min_trx_id()) { trx = 0; @@ -7097,9 +7093,8 @@ lock_sec_rec_read_check_and_lock( if the max trx id for the page >= min trx id for the trx list or a database recovery is running. */ - if ((page_get_max_trx_id(block->frame) >= trx_rw_min_trx_id() - || recv_recovery_is_on()) - && !page_rec_is_supremum(rec)) { + if (!page_rec_is_supremum(rec) + && page_get_max_trx_id(block->frame) >= trx_rw_min_trx_id()) { lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, index, offsets); diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 5ede7faf3ba..d9a04ba9393 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -2,6 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1279,7 +1280,7 @@ page_cur_insert_rec_low( == (ibool) !!page_is_comp(page)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || recv_recovery_is_on() + || index->is_dummy || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(!page_rec_is_supremum(current_rec)); @@ -1506,8 +1507,8 @@ page_cur_insert_rec_zip( ut_ad(page_is_comp(page)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index)) - || recv_recovery_is_on()); + || index->is_dummy + || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(!page_get_instant(page)); ut_ad(!page_cur_is_after_last(cursor)); #ifdef UNIV_ZIP_DEBUG @@ -1626,6 +1627,7 @@ page_cur_insert_rec_zip( because the MLOG_COMP_REC_INSERT should only be logged after a successful operation. */ ut_ad(!recv_recovery_is_on()); + ut_ad(!index->is_dummy); } else if (recv_recovery_is_on()) { /* This should be followed by MLOG_ZIP_PAGE_COMPRESS_NO_DATA, @@ -1967,6 +1969,8 @@ page_parse_copy_rec_list_to_created_page( page_t* page; page_zip_des_t* page_zip; + ut_ad(index->is_dummy); + if (ptr + 4 > end_ptr) { return(NULL); @@ -1992,8 +1996,7 @@ page_parse_copy_rec_list_to_created_page( page_copy_rec_list_end_to_created_page() which was logged by. page_copy_rec_list_to_created_page_write_log(). For other pages, this field must be zero-initialized. */ - ut_ad(!page_get_instant(block->frame) - || (page_is_root(block->frame) && index->is_dummy)); + ut_ad(!page_get_instant(block->frame) || page_is_root(block->frame)); while (ptr < rec_end) { ptr = page_cur_parse_insert_rec(TRUE, ptr, end_ptr, @@ -2320,8 +2323,8 @@ page_cur_delete_rec( ut_ad(!!page_is_comp(page) == dict_table_is_comp(index->table)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index)) - || recv_recovery_is_on()); + || index->is_dummy + || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(mtr == NULL || mtr->is_named_space(index->space)); /* The record must not be the supremum or infimum record. */ diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index c2ae828b1e0..60cc54f8a9b 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -715,7 +715,7 @@ row_upd_rec_in_place( ut_ad(index->is_instant()); break; case REC_STATUS_NODE_PTR: - if (recv_recovery_is_on() + if (index->is_dummy && fil_page_get_type(page_align(rec)) == FIL_PAGE_RTREE) { /* The function rtr_update_mbr_field_in_place() From 4cafd8e66f77798168aecd9bb239f97d7cd1ee3b Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 13 Jan 2018 01:26:06 +0300 Subject: [PATCH 43/82] rocksdb.information_schema testcase is not stable --- storage/rocksdb/mysql-test/rocksdb/t/disabled.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index 359d1d3e082..c2ca6efe46c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -67,6 +67,7 @@ lock_wait_timeout_stats: MDEV-13404 compact_deletes: MDEV-12663 : rocksdb.compact_deletes times out and causes other tests to fail blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api test fails +information_schema: MDEV-14372: unstable testcase ## ## Tests that fail for some other reason @@ -75,4 +76,3 @@ blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api tes mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs rpl_row_triggers : MariaRocks: requires GTIDs - From 0a63b50c7a2dcab6fe99885cfcbd4a35d322c63a Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Tue, 28 Nov 2017 16:34:31 +0400 Subject: [PATCH 44/82] Cleanup UT_LOW_PRIORITY_CPU/UT_RESUME_PRIORITY_CPU Server already has HMT_low/HMT_medium. --- config.h.cmake | 1 - configure.cmake | 11 ----------- include/my_cpu.h | 13 +++++++------ storage/innobase/include/ut0ut.h | 9 --------- storage/innobase/ut/ut0ut.cc | 5 +++-- 5 files changed, 10 insertions(+), 29 deletions(-) diff --git a/config.h.cmake b/config.h.cmake index 51abd4b11d5..31a563037a0 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -191,7 +191,6 @@ #cmakedefine HAVE_PREAD 1 #cmakedefine HAVE_PAUSE_INSTRUCTION 1 #cmakedefine HAVE_FAKE_PAUSE_INSTRUCTION 1 -#cmakedefine HAVE_HMT_PRIORITY_INSTRUCTION 1 #cmakedefine HAVE_RDTSCLL 1 #cmakedefine HAVE_READ_REAL_TIME 1 #cmakedefine HAVE_PTHREAD_ATTR_CREATE 1 diff --git a/configure.cmake b/configure.cmake index ffcb9ba83b9..40aca55dc47 100644 --- a/configure.cmake +++ b/configure.cmake @@ -783,17 +783,6 @@ IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC) } " HAVE_FAKE_PAUSE_INSTRUCTION) ENDIF() - IF (NOT HAVE_PAUSE_INSTRUCTION) - CHECK_C_SOURCE_COMPILES(" - #include - int main() - { - __ppc_set_ppr_low(); - __ppc_set_ppr_med(); - return 0; - } - " HAVE_HMT_PRIORITY_INSTRUCTION) - ENDIF() ENDIF() CHECK_SYMBOL_EXISTS(tcgetattr "termios.h" HAVE_TCGETATTR 1) diff --git a/include/my_cpu.h b/include/my_cpu.h index 856a8e9b04a..f2e26fca70c 100644 --- a/include/my_cpu.h +++ b/include/my_cpu.h @@ -23,17 +23,18 @@ The defines are the same ones used by the linux kernel */ -#if defined(__powerpc__) +#ifdef _ARCH_PWR8 +#include /* Very low priority */ -#define HMT_very_low() asm volatile("or 31,31,31") +#define HMT_very_low() __ppc_set_ppr_very_low() /* Low priority */ -#define HMT_low() asm volatile("or 1,1,1") +#define HMT_low() __ppc_set_ppr_low() /* Medium low priority */ -#define HMT_medium_low() asm volatile("or 6,6,6") +#define HMT_medium_low() __ppc_set_ppr_med_low() /* Medium priority */ -#define HMT_medium() asm volatile("or 2,2,2") +#define HMT_medium() __ppc_set_ppr_med() /* Medium high priority */ -#define HMT_medium_high() asm volatile("or 5,5,5") +#define HMT_medium_high() __ppc_set_ppr_med_high() /* High priority */ #define HMT_high() asm volatile("or 3,3,3") #else diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 352f5cce83d..c91e24ad314 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -60,15 +60,6 @@ typedef time_t ib_time_t; # define UT_COMPILER_BARRIER() #endif -#if defined(HAVE_HMT_PRIORITY_INSTRUCTION) -# include -# define UT_LOW_PRIORITY_CPU() __ppc_set_ppr_low() -# define UT_RESUME_PRIORITY_CPU() __ppc_set_ppr_med() -#else -# define UT_LOW_PRIORITY_CPU() ((void)0) -# define UT_RESUME_PRIORITY_CPU() ((void)0) -#endif - /*********************************************************************//** Delays execution for at most max_wait_us microseconds or returns earlier if cond becomes true. diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index 88c5c889c8d..25705cb0e2c 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -37,6 +37,7 @@ Created 5/11/1994 Heikki Tuuri #include "trx0trx.h" #include #include "log.h" +#include "my_cpu.h" #ifdef _WIN32 /*****************************************************************//** @@ -290,14 +291,14 @@ ut_delay( { ulint i; - UT_LOW_PRIORITY_CPU(); + HMT_low(); for (i = 0; i < delay * 50; i++) { MY_RELAX_CPU(); UT_COMPILER_BARRIER(); } - UT_RESUME_PRIORITY_CPU(); + HMT_medium(); } /*************************************************************//** From 09ef28abd7c3c80a80512bb4bba7f8e8f0194abe Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 13 Jan 2018 16:38:43 +0200 Subject: [PATCH 45/82] Fixed BUILD scripts - Removed extra set -x -v used for debugging - Fixed that that gcc version tests works for gcc 7 --- BUILD/FINISH.sh | 2 +- BUILD/check-cpu | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/BUILD/FINISH.sh b/BUILD/FINISH.sh index 1faf62a9b62..ea64183253d 100644 --- a/BUILD/FINISH.sh +++ b/BUILD/FINISH.sh @@ -14,7 +14,7 @@ # License along with this library; if not, write to the Free # Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, # MA 02110-1301, USA -set -x -v + cflags="$c_warnings $extra_flags $EXTRA_FLAGS $EXTRA_CFLAGS" cxxflags="$cxx_warnings $base_cxxflags $extra_flags $EXTRA_FLAGS $EXTRA_CXXFLAGS" extra_configs="$extra_configs $local_infile_configs $EXTRA_CONFIGS" diff --git a/BUILD/check-cpu b/BUILD/check-cpu index ad8816dc421..c1c85cfd908 100755 --- a/BUILD/check-cpu +++ b/BUILD/check-cpu @@ -40,6 +40,12 @@ check_compiler_cpu_flags () { cc_major=$1 cc_minor=$2 cc_patch=$3 + if test -z "$cc_minor"; then + cc_minor="0"; + fi + if test -z "$cc_patch"; then + cc_minor="0"; + fi cc_comp=`expr $cc_major '*' 100 '+' $cc_minor` fi From fc65577873967ecbba93b75260cc54275f5900d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 13 Jan 2018 18:04:03 +0200 Subject: [PATCH 46/82] MDEV-14887 On a 32-bit system, MariaDB 10.2 mishandles data file sizes exceeding 4GiB This is a regression that was introduced in MySQL 5.7.6 in https://github.com/mysql/mysql-server/commit/19855664de0245ff24e0753dc82723fc4e2fb7a5 fil_node_open_file(): Use proper 64-bit arithmetics for truncating size_bytes to a multiple of a file extent size. --- storage/innobase/fil/fil0fil.cc | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 3b496e3959f..86c0499eca4 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -566,7 +566,6 @@ bool fil_node_open_file( fil_node_t* node) { - os_offset_t size_bytes; bool success; bool read_only_mode; fil_space_t* space = node->space; @@ -611,7 +610,7 @@ retry: return(false); } - size_bytes = os_file_get_size(node->handle); + os_offset_t size_bytes = os_file_get_size(node->handle); ut_a(size_bytes != (os_offset_t) -1); ut_a(space->purpose != FIL_TYPE_LOG); @@ -694,20 +693,17 @@ retry: space->free_len = free_len; if (first_time_open) { - ulint extent_size; - - extent_size = psize * FSP_EXTENT_SIZE; - - /* After apply-incremental, tablespaces are not extended - to a whole megabyte. Do not cut off valid data. */ - /* Truncate the size to a multiple of extent size. */ - if (size_bytes >= extent_size) { - size_bytes = ut_2pow_round(size_bytes, - extent_size); + ulint mask = psize * FSP_EXTENT_SIZE - 1; + + if (size_bytes <= mask) { + /* .ibd files start smaller than an + extent size. Do not truncate valid data. */ + } else { + size_bytes &= ~os_offset_t(mask); } - node->size = static_cast(size_bytes / psize); + node->size = ulint(size_bytes / psize); space->size += node->size; } } From 68e5d6a9416b2923e78123cbc67e1755b8a21806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 13 Jan 2018 20:27:46 +0200 Subject: [PATCH 47/82] Do not truncate integers on 32-bit systems in Rows_event_tracker commit 3dc3ab1a3048484910ca8acccaf76c71b080e533 introduced Rows_event_tracker, using a mismatch of size_t (the native register width) and my_off_t (the file offset width, usually 64 bits). Use my_off_t both in member fields and member functions. --- sql/rpl_mi.h | 4 ++-- sql/slave.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index b304e45f86a..b81c1fb398a 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -139,11 +139,11 @@ typedef struct st_rows_event_tracker my_off_t first_seen; my_off_t last_seen; bool stmt_end_seen; - void update(const char* file_name, size_t pos, + void update(const char* file_name, my_off_t pos, const char* buf, const Format_description_log_event *fdle); void reset(); - bool check_and_report(const char* file_name, size_t pos); + bool check_and_report(const char* file_name, my_off_t pos); } Rows_event_tracker; /***************************************************************************** diff --git a/sql/slave.cc b/sql/slave.cc index e5c502c2de5..d124b15e47c 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -7650,7 +7650,7 @@ void Rows_event_tracker::reset() well as the end-of-statement status of the last one. */ -void Rows_event_tracker::update(const char* file_name, size_t pos, +void Rows_event_tracker::update(const char* file_name, my_off_t pos, const char* buf, const Format_description_log_event *fdle) { @@ -7675,7 +7675,7 @@ void Rows_event_tracker::update(const char* file_name, size_t pos, false otherwise. */ bool Rows_event_tracker::check_and_report(const char* file_name, - size_t pos) + my_off_t pos) { if (last_seen) { From 72136ae75c69cee5230e7be286cf12663d3d15ef Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sun, 14 Jan 2018 19:50:45 +0300 Subject: [PATCH 48/82] Compilation speed (#546) Speed up compilation Standard C++ headers contribute a lot to compilation time. Avoid algorithm and sstream in frequently used headers. --- include/my_dbug.h | 6 +++++- sql/item_func.cc | 2 +- sql/item_sum.cc | 4 ++-- sql/key.cc | 19 ++++++++----------- sql/log.cc | 2 +- sql/log_event.cc | 6 ++---- sql/mdl.h | 5 +---- storage/perfschema/pfs.cc | 3 +-- storage/perfschema/table_threads.cc | 4 ++-- 9 files changed, 23 insertions(+), 28 deletions(-) diff --git a/include/my_dbug.h b/include/my_dbug.h index ba9e8a025d7..003290afb96 100644 --- a/include/my_dbug.h +++ b/include/my_dbug.h @@ -195,10 +195,14 @@ void debug_sync_point(const char* lock_name, uint lock_timeout); #ifdef __cplusplus } +/* + DBUG_LOG() was initially intended for InnoDB. To be able to use it elsewhere + one should #include . We intentially avoid including it here to save + compilation time. +*/ # ifdef DBUG_OFF # define DBUG_LOG(keyword, v) do {} while (0) # else -# include # define DBUG_LOG(keyword, v) do { \ if (_db_pargs_(__LINE__, keyword)) { \ std::ostringstream _db_s; _db_s << v; \ diff --git a/sql/item_func.cc b/sql/item_func.cc index b1970be2b29..de740cbcece 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2889,7 +2889,7 @@ bool Item_func_min_max::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) ltime->hour+= (ltime->month * 32 + ltime->day) * 24; ltime->year= ltime->month= ltime->day= 0; if (adjust_time_range_with_warn(ltime, - std::min(decimals, TIME_SECOND_PART_DIGITS))) + MY_MIN(decimals, TIME_SECOND_PART_DIGITS))) return (null_value= true); } diff --git a/sql/item_sum.cc b/sql/item_sum.cc index f5ea2a5df93..94569aa66c9 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -2257,7 +2257,7 @@ bool Item_sum_bit::remove_as_window(ulonglong value) } // Prevent overflow; - num_values_added = std::min(num_values_added, num_values_added - 1); + num_values_added = MY_MIN(num_values_added, num_values_added - 1); set_bits_from_counters(); return 0; } @@ -2270,7 +2270,7 @@ bool Item_sum_bit::add_as_window(ulonglong value) bit_counters[i]+= (value & (1ULL << i)) ? 1 : 0; } // Prevent overflow; - num_values_added = std::max(num_values_added, num_values_added + 1); + num_values_added = MY_MAX(num_values_added, num_values_added + 1); set_bits_from_counters(); return 0; } diff --git a/sql/key.cc b/sql/key.cc index cdf5c8c4a74..af473d85847 100644 --- a/sql/key.cc +++ b/sql/key.cc @@ -21,9 +21,6 @@ #include "key.h" // key_rec_cmp #include "field.h" // Field -using std::min; -using std::max; - /* Search after a key that starts with 'field' @@ -135,7 +132,7 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, Don't copy data for null values The -1 below is to subtract the null byte which is already handled */ - length= min(key_length, key_part->store_length-1); + length= MY_MIN(key_length, key_part->store_length-1); if (with_zerofill) bzero((char*) to_key, length); continue; @@ -145,7 +142,7 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, key_part->key_part_flag & HA_VAR_LENGTH_PART) { key_length-= HA_KEY_BLOB_LENGTH; - length= min(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); uint bytes= key_part->field->get_key_image(to_key, length, Field::itRAW); if (with_zerofill && bytes < length) bzero((char*) to_key + bytes, length - bytes); @@ -153,7 +150,7 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, } else { - length= min(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); Field *field= key_part->field; CHARSET_INFO *cs= field->charset(); uint bytes= field->get_key_image(to_key, length, Field::itRAW); @@ -205,7 +202,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, Don't copy data for null bytes The -1 below is to subtract the null byte which is already handled */ - length= min(key_length, key_part->store_length-1); + length= MY_MIN(key_length, key_part->store_length-1); continue; } } @@ -247,7 +244,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, my_ptrdiff_t ptrdiff= to_record - field->table->record[0]; field->move_field_offset(ptrdiff); key_length-= HA_KEY_BLOB_LENGTH; - length= min(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); old_map= dbug_tmp_use_all_columns(field->table, field->table->write_set); field->set_key_image(from_key, length); dbug_tmp_restore_column_map(field->table->write_set, old_map); @@ -256,7 +253,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, } else { - length= min(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); /* skip the byte with 'uneven' bits, if used */ memcpy(to_record + key_part->offset, from_key + used_uneven_bits , (size_t) length - used_uneven_bits); @@ -314,7 +311,7 @@ bool key_cmp_if_same(TABLE *table,const uchar *key,uint idx,uint key_length) return 1; continue; } - length= min((uint) (key_end-key), store_length); + length= MY_MIN((uint) (key_end-key), store_length); if (!(key_part->key_type & (FIELDFLAG_NUMBER+FIELDFLAG_BINARY+ FIELDFLAG_PACK))) { @@ -392,7 +389,7 @@ void field_unpack(String *to, Field *field, const uchar *rec, uint max_length, tmp.length(charpos); } if (max_length < field->pack_length()) - tmp.length(min(tmp.length(),max_length)); + tmp.length(MY_MIN(tmp.length(),max_length)); ErrConvString err(&tmp); to->append(err.ptr()); } diff --git a/sql/log.cc b/sql/log.cc index 23873922368..f1a0834217a 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -7044,7 +7044,7 @@ int MYSQL_BIN_LOG::write_cache(THD *thd, IO_CACHE *cache) int4store(ev + EVENT_LEN_OFFSET, ev_len + writer.checksum_len); writer.remains= ev_len; - if (writer.write(ev, std::min(ev_len, length - hdr_offs))) + if (writer.write(ev, MY_MIN(ev_len, length - hdr_offs))) DBUG_RETURN(ER_ERROR_ON_WRITE); /* next event header at ... */ diff --git a/sql/log_event.cc b/sql/log_event.cc index bd6d325b3ef..d80f45dffc1 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -55,8 +55,6 @@ #define my_b_write_string(A, B) my_b_write((A), (uchar*)(B), (uint) (sizeof(B) - 1)) -using std::max; - /** BINLOG_CHECKSUM variable. */ @@ -1777,8 +1775,8 @@ int Log_event::read_log_event(IO_CACHE* file, String* packet, if (data_len < LOG_EVENT_MINIMAL_HEADER_LEN) DBUG_RETURN(LOG_READ_BOGUS); - if (data_len > max(max_allowed_packet, - opt_binlog_rows_event_max_size + MAX_LOG_EVENT_HEADER)) + if (data_len > MY_MAX(max_allowed_packet, + opt_binlog_rows_event_max_size + MAX_LOG_EVENT_HEADER)) DBUG_RETURN(LOG_READ_TOO_LARGE); if (likely(data_len > LOG_EVENT_MINIMAL_HEADER_LEN)) diff --git a/sql/mdl.h b/sql/mdl.h index 97216b8e7b1..2aeaef89f26 100644 --- a/sql/mdl.h +++ b/sql/mdl.h @@ -21,8 +21,6 @@ #include #include -#include - class THD; class MDL_context; @@ -373,8 +371,7 @@ public: character set is utf-8, we can safely assume that no character starts with a zero byte. */ - using std::min; - return memcmp(m_ptr, rhs->m_ptr, min(m_length, rhs->m_length)); + return memcmp(m_ptr, rhs->m_ptr, MY_MIN(m_length, rhs->m_length)); } MDL_key(const MDL_key *rhs) diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index f781a83b324..624c7f4697b 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -41,7 +41,6 @@ #include "sp_head.h" #include "pfs_digest.h" -using std::min; /** @page PAGE_PERFORMANCE_SCHEMA The Performance Schema main page MySQL PERFORMANCE_SCHEMA implementation. @@ -2022,7 +2021,7 @@ static void set_thread_account_v1(const char *user, int user_len, DBUG_ASSERT((host != NULL) || (host_len == 0)); DBUG_ASSERT(host_len >= 0); - host_len= min(host_len, sizeof(pfs->m_hostname)); + host_len= MY_MIN(host_len, static_cast(sizeof(pfs->m_hostname))); if (unlikely(pfs == NULL)) return; diff --git a/storage/perfschema/table_threads.cc b/storage/perfschema/table_threads.cc index 5c78b567b8c..211865c0f63 100644 --- a/storage/perfschema/table_threads.cc +++ b/storage/perfschema/table_threads.cc @@ -257,8 +257,8 @@ int table_threads::read_row_values(TABLE *table, changed to less than or equal to 64 characters. */ set_field_varchar_utf8(f, m_row.m_processlist_state_ptr, - std::min(m_row.m_processlist_state_length, - f->char_length())); + MY_MIN(m_row.m_processlist_state_length, + f->char_length())); } else f->set_null(); From 3d798be1d481835b60a811148a10ea14c37d5674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 10:49:03 +0200 Subject: [PATCH 49/82] MDEV-14655 Assertion `!fts_index' failed in prepare_inplace_alter_table_dict MariaDB inherits the MySQL limitation that ALGORITHM=INPLACE cannot create more than one FULLTEXT INDEX at a time. As part of the MDEV-11369 Instant ADD COLUMN refactoring, MariaDB 10.3.2 accidentally stopped enforcing the restriction. Actually, it is a bug in MySQL 5.6 and MariaDB 10.0 that an ALTER TABLE statement with multiple ADD FULLTEXT INDEX but without explicit ALGORITHM=INPLACE would return in an error message, rather than executing the operation with ALGORITHM=COPY. ha_innobase::check_if_supported_inplace_alter(): Enforce the restriction on multiple FULLTEXT INDEX. prepare_inplace_alter_table_dict(): Replace some code with debug assertions. A "goto error_handled" at this point would result in another error, because the reference count of ctx->new_table would be 0. --- .../suite/innodb_fts/r/innodb-fts-ddl.result | 14 +++-- .../suite/innodb_fts/t/innodb-fts-ddl.test | 13 ++--- storage/innobase/handler/handler0alter.cc | 51 +++++++------------ 3 files changed, 35 insertions(+), 43 deletions(-) diff --git a/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result b/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result index 983f254cf90..cd7d8f03923 100644 --- a/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result +++ b/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result @@ -162,11 +162,15 @@ INSERT INTO articles (FTS_DOC_ID, title, body) VALUES (14,'1001 MySQL Tricks','1. Never run mysqld as root. 2. ...'), (19, 'MySQL vs. YourSQL','In the following database comparison ...'), (20, 'MySQL Security','When configured properly, MySQL ...'); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title), -ADD FULLTEXT INDEX idx5 (title); -ERROR HY000: InnoDB presently supports one FULLTEXT index creation at a time -CREATE FULLTEXT INDEX idx on articles (title); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title); +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), +ADD FULLTEXT INDEX idx3 (title), ALGORITHM=INPLACE; +ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: InnoDB presently supports one FULLTEXT index creation at a time. Try ALGORITHM=COPY +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), +ADD FULLTEXT INDEX idx3 (title); +affected rows: 6 +info: Records: 6 Duplicates: 0 Warnings: 1 +Warnings: +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release ALTER TABLE articles ADD INDEX t20 (title(20)), LOCK=NONE; ALTER TABLE articles DROP INDEX t20; INSERT INTO articles (FTS_DOC_ID, title, body) VALUES diff --git a/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test b/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test index 10dc1462c98..23065a97002 100644 --- a/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test +++ b/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test @@ -195,12 +195,13 @@ INSERT INTO articles (FTS_DOC_ID, title, body) VALUES (19, 'MySQL vs. YourSQL','In the following database comparison ...'), (20, 'MySQL Security','When configured properly, MySQL ...'); ---error ER_INNODB_FT_LIMIT -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title), - ADD FULLTEXT INDEX idx5 (title); - -CREATE FULLTEXT INDEX idx on articles (title); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title); +--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), + ADD FULLTEXT INDEX idx3 (title), ALGORITHM=INPLACE; +--enable_info +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), + ADD FULLTEXT INDEX idx3 (title); +--disable_info ALTER TABLE articles ADD INDEX t20 (title(20)), LOCK=NONE; ALTER TABLE articles DROP INDEX t20; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 7d4b15fc40e..55962e1c6d1 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1162,6 +1162,7 @@ next_column: /* If the table already contains fulltext indexes, refuse to rebuild the table natively altogether. */ if (m_prebuilt->table->fts) { +cannot_create_many_fulltext_index: ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_INNODB_FT_LIMIT); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); @@ -1191,6 +1192,7 @@ next_column: We could also do ADD SPATIAL INDEX by implementing row_log_apply() for it. */ + bool add_fulltext = false; for (uint i = 0; i < ha_alter_info->index_add_count; i++) { const KEY* key = @@ -1202,16 +1204,18 @@ next_column: | HA_PACK_KEY | HA_GENERATED_KEY | HA_BINARY_PACK_KEY))); + if (add_fulltext) { + goto cannot_create_many_fulltext_index; + } + add_fulltext = true; ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_FTS); online = false; - break; } - if (key->flags & HA_SPATIAL) { + if (online && (key->flags & HA_SPATIAL)) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_GIS); online = false; - break; } } } @@ -5291,35 +5295,18 @@ new_clustered_failed: ctx->prepare_instant(); } - if (!ctx->is_instant()) { - if (num_fts_index > 1) { - my_error(ER_INNODB_FT_LIMIT, MYF(0)); - goto error_handled; - } - - if (!ctx->online) { - /* This is not an online operation (LOCK=NONE). */ - } else if (ctx->add_autoinc == ULINT_UNDEFINED - && num_fts_index == 0 - && (!innobase_need_rebuild(ha_alter_info, old_table) - || !innobase_fulltext_exist(altered_table))) { - /* InnoDB can perform an online operation - (LOCK=NONE). */ - } else { - size_t query_length; - /* This should have been blocked in - check_if_supported_inplace_alter(). */ - ut_ad(0); - my_error(ER_NOT_SUPPORTED_YET, MYF(0), - innobase_get_stmt_unsafe( - ctx->prebuilt->trx->mysql_thd, - &query_length)); - goto error_handled; - } - } - if (ctx->need_rebuild()) { not_instant_add_column: + DBUG_ASSERT(ctx->need_rebuild()); + DBUG_ASSERT(!ctx->is_instant()); + DBUG_ASSERT(num_fts_index <= 1); + DBUG_ASSERT(!ctx->online || num_fts_index == 0); + DBUG_ASSERT(!ctx->online + || ctx->add_autoinc == ULINT_UNDEFINED); + DBUG_ASSERT(!ctx->online + || !innobase_need_rebuild(ha_alter_info, old_table) + || !innobase_fulltext_exist(altered_table)); + uint32_t key_id = FIL_DEFAULT_ENCRYPTION_KEY; fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT; @@ -5419,7 +5406,7 @@ new_table_failed: ut_ad(index->trx_id == ctx->trx->id); if (index->type & DICT_FTS) { - DBUG_ASSERT(num_fts_index); + DBUG_ASSERT(num_fts_index == 1); DBUG_ASSERT(!fts_index); DBUG_ASSERT(index->type == DICT_FTS); fts_index = ctx->add_index[a]; @@ -5504,7 +5491,7 @@ error_handling_drop_uncached: /* If ADD INDEX with LOCK=NONE has been requested, allocate a modification log. */ if (index->type & DICT_FTS) { - DBUG_ASSERT(num_fts_index); + DBUG_ASSERT(num_fts_index == 1); DBUG_ASSERT(!fts_index); DBUG_ASSERT(index->type == DICT_FTS); fts_index = ctx->add_index[a]; From ec062c618186681504e79e6f568f98406020ab01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 15:26:02 +0200 Subject: [PATCH 50/82] MDEV-12121 follow-up: Unbreak the WITH_INNODB_AHI=OFF build --- storage/innobase/buf/buf0lru.cc | 12 ++++++++---- storage/innobase/dict/dict0dict.cc | 8 ++++++-- storage/innobase/fil/fil0fil.cc | 20 ++++++++++++++++---- storage/innobase/handler/ha_xtradb.h | 4 +++- storage/innobase/include/buf0lru.h | 12 +++++++----- storage/innobase/include/fil0fil.h | 10 +++++++--- storage/innobase/include/ha0ha.h | 3 ++- 7 files changed, 49 insertions(+), 20 deletions(-) diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 10ee106137d..2d68b8e9ccd 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -696,22 +696,26 @@ buf_flush_dirty_pages( /** Empty the flush list for all pages belonging to a tablespace. @param[in] id tablespace identifier @param[in] observer flush observer, - or NULL if nothing is to be written -@param[in] drop_ahi whether to drop the adaptive hash index */ + or NULL if nothing is to be written */ void buf_LRU_flush_or_remove_pages( ulint id, - FlushObserver* observer, - bool drop_ahi) + FlushObserver* observer +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ) { /* Pages in the system tablespace must never be discarded. */ ut_ad(id || observer); for (ulint i = 0; i < srv_buf_pool_instances; i++) { buf_pool_t* buf_pool = buf_pool_from_array(i); +#ifdef BTR_CUR_HASH_ADAPT if (drop_ahi) { buf_LRU_drop_page_hash_for_tablespace(buf_pool, id); } +#endif /* BTR_CUR_HASH_ADAPT */ buf_flush_dirty_pages(buf_pool, id, observer); } diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 49c60e5df56..b93eb4d6b17 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1641,7 +1641,11 @@ dict_table_rename_in_cache( return(DB_OUT_OF_MEMORY); } - fil_delete_tablespace(table->space, true); + fil_delete_tablespace(table->space +#ifdef BTR_CUR_HASH_ADAPT + , true +#endif /* BTR_CUR_HASH_ADAPT */ + ); /* Delete any temp file hanging around. */ if (os_file_status(filepath, &exists, &ftype) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 86c0499eca4..9e31cb8bd81 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2960,10 +2960,14 @@ fil_table_accessible(const dict_table_t* table) /** Delete a tablespace and associated .ibd file. @param[in] id tablespace identifier -@param[in] drop_ahi whether to drop the adaptive hash index @return DB_SUCCESS or error */ dberr_t -fil_delete_tablespace(ulint id, bool drop_ahi) +fil_delete_tablespace( + ulint id +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ) { char* path = 0; fil_space_t* space = 0; @@ -3006,7 +3010,11 @@ fil_delete_tablespace(ulint id, bool drop_ahi) To deal with potential read requests, we will check the ::stop_new_ops flag in fil_io(). */ - buf_LRU_flush_or_remove_pages(id, NULL, drop_ahi); + buf_LRU_flush_or_remove_pages(id, NULL +#ifdef BTR_CUR_HASH_ADAPT + , drop_ahi +#endif /* BTR_CUR_HASH_ADAPT */ + ); /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ @@ -3286,7 +3294,11 @@ fil_discard_tablespace( { dberr_t err; - switch (err = fil_delete_tablespace(id, true)) { + switch (err = fil_delete_tablespace(id +#ifdef BTR_CUR_HASH_ADAPT + , true +#endif /* BTR_CUR_HASH_ADAPT */ + )) { case DB_SUCCESS: break; diff --git a/storage/innobase/handler/ha_xtradb.h b/storage/innobase/handler/ha_xtradb.h index 09628b7ec48..d2d1f361382 100644 --- a/storage/innobase/handler/ha_xtradb.h +++ b/storage/innobase/handler/ha_xtradb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Copyright (c) 2009, Percona Inc. Portions of this file contain modifications contributed and copyrighted @@ -957,7 +957,9 @@ innodb_check_deprecated(void) #ifdef HA_XTRADB_SYSVARS /* XtraDB compatibility system variables */ +#ifdef BTR_CUR_HASH_ADAPT MYSQL_SYSVAR(adaptive_hash_index_partitions), +#endif /* BTR_CUR_HASH_ADAPT */ MYSQL_SYSVAR(buffer_pool_populate), #if defined UNIV_DEBUG || defined UNIV_PERF_DEBUG MYSQL_SYSVAR(cleaner_eviction_factor), diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h index 7b739fc0332..f6a7695a2b5 100644 --- a/storage/innobase/include/buf0lru.h +++ b/storage/innobase/include/buf0lru.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,13 +53,15 @@ These are low-level functions /** Empty the flush list for all pages belonging to a tablespace. @param[in] id tablespace identifier @param[in,out] observer flush observer, - or NULL if nothing is to be written -@param[in] drop_ahi whether to drop the adaptive hash index */ + or NULL if nothing is to be written */ void buf_LRU_flush_or_remove_pages( ulint id, - FlushObserver* observer, - bool drop_ahi = false); + FlushObserver* observer +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi = false /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ); #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG /********************************************************************//** diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9fa507c2114..c2152ce11d0 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -931,10 +931,14 @@ fil_table_accessible(const dict_table_t* table) /** Delete a tablespace and associated .ibd file. @param[in] id tablespace identifier -@param[in] drop_ahi whether to drop the adaptive hash index @return DB_SUCCESS or error */ dberr_t -fil_delete_tablespace(ulint id, bool drop_ahi = false); +fil_delete_tablespace( + ulint id +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi = false /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ); /** Truncate the tablespace to needed size. @param[in] space_id id of tablespace to truncate diff --git a/storage/innobase/include/ha0ha.h b/storage/innobase/include/ha0ha.h index ca4cb0a5f8f..f5be654f490 100644 --- a/storage/innobase/include/ha0ha.h +++ b/storage/innobase/include/ha0ha.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -210,7 +211,7 @@ struct ha_node_t { }; #endif /* BTR_CUR_HASH_ADAPT */ -#ifdef UNIV_DEBUG +#if defined UNIV_DEBUG && defined BTR_CUR_HASH_ADAPT /********************************************************************//** Assert that the synchronization object in a hash operation involving possible change in the hash table is held. From 850702da6bed7e18145be7688d95225241ac29bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 15:37:22 +0200 Subject: [PATCH 51/82] MDEV-13626: Merge InnoDB test cases from MySQL 5.7 (part 6) innodb.truncate_inject: Replacement for innodb_zip.wl6501_error_1 Note: unlike MySQL, in some cases TRUNCATE does not return an error in MariaDB. This should be fixed in the scope of MDEV-13564 or similar. --- .../suite/innodb/r/truncate_inject.result | 114 +++++++++ .../suite/innodb/t/truncate_inject.test | 97 ++++++++ .../include/innodb_wl6501_error.inc | 226 ------------------ 3 files changed, 211 insertions(+), 226 deletions(-) create mode 100644 mysql-test/suite/innodb/r/truncate_inject.result create mode 100644 mysql-test/suite/innodb/t/truncate_inject.test delete mode 100644 mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc diff --git a/mysql-test/suite/innodb/r/truncate_inject.result b/mysql-test/suite/innodb/r/truncate_inject.result new file mode 100644 index 00000000000..5ec532a0f83 --- /dev/null +++ b/mysql-test/suite/innodb/r/truncate_inject.result @@ -0,0 +1,114 @@ +SET @save_dbug = @@SESSION.debug_dbug; +call mtr.add_suppression("InnoDB: Flagged corruption of .* in table `test`\\.`t` in TRUNCATE TABLE"); +# 1. Error in assigning undo logs for truncate action +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_assigning_undo_log'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +1 1.1 a +2 2.2 b +3 3.3 c +# 2. Error while preparing for truncate +SET debug_dbug = '+d,ib_err_trunc_preparing_for_truncate'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +1 1.1 a +2 2.2 b +3 3.3 c +# 3. Error while dropping/creating indexes +SET debug_dbug = '+d,ib_err_trunc_drop_index'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check Warning InnoDB: Index PRIMARY is marked as corrupted +test.t check error Corrupt +select * from t; +Got one of the listed errors +drop table t; +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_create_index'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check Warning InnoDB: Index PRIMARY is marked as corrupted +test.t check error Corrupt +select * from t; +Got one of the listed errors +drop table t; +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +drop table t; +# 4. Error while completing truncate of table involving FTS +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), +(2, 2.2, 'innodb is part of mysql'), +(3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +drop table t; +# 5. Error while updating sys-tables +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), +(2, 2.2, 'innodb is part of mysql'), +(3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t order by i; +i f c +drop table t; diff --git a/mysql-test/suite/innodb/t/truncate_inject.test b/mysql-test/suite/innodb/t/truncate_inject.test new file mode 100644 index 00000000000..35e516324bb --- /dev/null +++ b/mysql-test/suite/innodb/t/truncate_inject.test @@ -0,0 +1,97 @@ +# This test is based on innodb_zip.wl6501_error_1 in MySQL 5.7. + +--source include/have_innodb.inc +--source include/innodb_row_format.inc +--source include/have_debug.inc + +SET @save_dbug = @@SESSION.debug_dbug; + +call mtr.add_suppression("InnoDB: Flagged corruption of .* in table `test`\\.`t` in TRUNCATE TABLE"); + +--echo # 1. Error in assigning undo logs for truncate action +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +# +SET debug_dbug = '+d,ib_err_trunc_assigning_undo_log'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +select * from t; + +--echo # 2. Error while preparing for truncate +SET debug_dbug = '+d,ib_err_trunc_preparing_for_truncate'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +select * from t; + +--echo # 3. Error while dropping/creating indexes +SET debug_dbug = '+d,ib_err_trunc_drop_index'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +--error ER_TABLE_CORRUPT,ER_GET_ERRNO +select * from t; +drop table t; + +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; + +SET debug_dbug = '+d,ib_err_trunc_create_index'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +--error ER_TABLE_CORRUPT,ER_GET_ERRNO +select * from t; +drop table t; + +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; + +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t; +drop table t; + +--echo # 4. Error while completing truncate of table involving FTS +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), + (2, 2.2, 'innodb is part of mysql'), + (3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t; +drop table t; + +--echo # 5. Error while updating sys-tables +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), + (2, 2.2, 'innodb is part of mysql'), + (3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t order by i; +drop table t; diff --git a/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc b/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc deleted file mode 100644 index 424608f251b..00000000000 --- a/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc +++ /dev/null @@ -1,226 +0,0 @@ -# -# WL#6501: make truncate table atomic -# - ---source include/have_innodb.inc ---source include/have_debug.inc - ---disable_query_log -# suppress expected warnings -call mtr.add_suppression("Unable to truncate FTS index for table"); -call mtr.add_suppression("Unable to assign a new identifier to table " - "`.*`\.`.*` after truncating it"); -call mtr.add_suppression("Flagged corruption of .* in table " - "`.*`\.`.*` in TRUNCATE TABLE"); -call mtr.add_suppression("Parent table of FTS auxiliary table " - ".*\/.* not found"); ---enable_query_log -################################################################################ -# -# Will test following scenarios: -# 1. Error in assigning undo logs for truncate action. -# 2. Error while preparing for truncate. -# 3. Error while dropping/creating indexes. -# 4. Error while completing truncate of table involving FTS. -# 5. Error while updating sys-tables. -# -################################################################################ - -#----------------------------------------------------------------------------- -# -# create test-bed -# -let $per_table = `select @@innodb_file_per_table`; -let $format = `select @@innodb_file_format`; - -eval set global innodb_file_per_table = on; -let $WL6501_TMP_DIR = `select @@tmpdir`; -let $WL6501_DATA_DIR = `select @@datadir`; -set innodb_strict_mode=off; - -#----------------------------------------------------------------------------- -# -# 1. Error in assigning undo logs for truncate action. -# ---echo "1. Error in assigning undo logs for truncate action." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_assigning_undo_log"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_assigning_undo_log"; -# -#check table t; -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 2. Error while preparing for truncate. -# ---echo "2. Error while preparing for truncate." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_preparing_for_truncate"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_preparing_for_truncate"; -# -#check table t; -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 3. Error while dropping/creating indexes -# ---echo "3. Error while dropping/creating indexes" -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_drop_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_drop_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; -# -# -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_create_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_create_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; -# -# -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create temporary table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_temp_recreate_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_temp_recreate_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 4. Error while completing truncate of table involving FTS. -# ---echo "4. Error while completing truncate of table involving FTS." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t (i int, f float, c char(100), - primary key pk(i), index fk(f), fulltext index ck(c)) - engine=innodb row_format=$wl6501_row_fmt - key_block_size=$wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'mysql is now oracle company'), - (2, 2.2, 'innodb is part of mysql'), - (3, 3.3, 'innodb is default storage engine of mysql'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_during_fts_trunc"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_during_fts_trunc"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 5. Error while updating sys-tables. -# ---echo "5. Error while updating sys-tables." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t (i int, f float, c char(100), - primary key pk(i), index fk(f), fulltext index ck(c)) - engine=innodb row_format=$wl6501_row_fmt - key_block_size=$wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'mysql is now oracle company'), - (2, 2.2, 'innodb is part of mysql'), - (3, 3.3, 'innodb is default storage engine of mysql'); -select * from t order by i; -#check table t; -# -set session debug = "+d,ib_err_trunc_during_sys_table_update"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_during_sys_table_update"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t order by i; -drop table t; - -#----------------------------------------------------------------------------- -# -# remove test-bed -# -eval set global innodb_file_per_table = $per_table; From 85aea5a12b711c3d94e1bbeb737f267590a0e3aa Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 15 Jan 2018 16:50:18 +0300 Subject: [PATCH 52/82] Update .result for rocksdb.rpl_row_triggers (not the whole test works yet) --- .../rocksdb/r/rpl_row_triggers.result | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result b/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result index 1d3cd7db641..69acc4a92e8 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result @@ -1,12 +1,12 @@ include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] # Test of row replication with triggers on the slave side +connection master; CREATE TABLE t1 (C1 CHAR(1) primary key, C2 CHAR(1)); SELECT * FROM t1; C1 C2 +connection slave; +connection slave; SET @old_slave_exec_mode= @@global.slave_exec_mode; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET @@global.slave_exec_mode= IDEMPOTENT; @@ -38,8 +38,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # INSERT triggers test insert into t1 values ('a','b'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -48,8 +51,11 @@ i0 1 a i1 1 a u0 0 u1 0 +connection master; # UPDATE triggers test update t1 set C1= 'd'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -58,8 +64,11 @@ i0 1 a i1 1 a u0 1 a d u1 1 a d +connection master; # DELETE triggers test delete from t1 where C1='d'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 1 d @@ -78,7 +87,10 @@ i0 2 0 i1 2 0 u0 1 a d u1 1 a d +connection master; insert into t1 values ('0','1'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 1 d @@ -90,32 +102,44 @@ u1 2 0 0 # INSERT triggers which cause also DELETE test # (insert duplicate row in table referenced by foreign key) insert into t1 values ('1','1'); +connection master; drop table if exists t1; +connection slave; +connection slave; SET @@global.slave_exec_mode= @old_slave_exec_mode; SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; drop table t2; +connection master; CREATE TABLE t1 (i INT); CREATE TABLE t2 (i INT); +connection slave; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET GLOBAL slave_run_triggers_for_rbr=YES; CREATE TRIGGER tr AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t2 VALUES (new.i); +connection master; BEGIN; INSERT INTO t1 VALUES (1); INSERT INTO t1 VALUES (2); COMMIT; +connection slave; select * from t2; i 1 2 SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; +connection master; drop tables t2,t1; +connection slave; # Triggers on slave do not work if master has some +connection master; CREATE TABLE t1 (C1 CHAR(1) primary key, C2 CHAR(1)); SELECT * FROM t1; C1 C2 create trigger t1_dummy before delete on t1 for each row set @dummy= 1; +connection slave; +connection slave; SET @old_slave_exec_mode= @@global.slave_exec_mode; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET @@global.slave_exec_mode= IDEMPOTENT; @@ -145,8 +169,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # INSERT triggers test insert into t1 values ('a','b'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -155,8 +182,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # UPDATE triggers test update t1 set C1= 'd'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -165,8 +195,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # DELETE triggers test delete from t1 where C1='d'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -185,7 +218,10 @@ i0 1 0 i1 1 0 u0 0 u1 0 +connection master; insert into t1 values ('0','1'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -197,22 +233,30 @@ u1 0 # INSERT triggers which cause also DELETE test # (insert duplicate row in table referenced by foreign key) insert into t1 values ('1','1'); +connection master; drop table if exists t1; +connection slave; +connection slave; SET @@global.slave_exec_mode= @old_slave_exec_mode; SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; drop table t2; # # MDEV-5513: Trigger is applied to the rows after first one # +connection master; create table t1 (a int, b int); create table tlog (a int auto_increment primary key); set sql_log_bin=0; create trigger tr1 after insert on t1 for each row insert into tlog values (null); set sql_log_bin=1; +connection slave; +connection slave; set @slave_run_triggers_for_rbr.saved = @@slave_run_triggers_for_rbr; set global slave_run_triggers_for_rbr=1; create trigger tr2 before insert on t1 for each row set new.b = new.a; +connection master; insert into t1 values (1,10),(2,20),(3,30); +connection slave; select * from t1; a b 1 10 From 4794e5b091c384e4d94b3c5f3c1629cba388d9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 16:19:46 +0200 Subject: [PATCH 53/82] Fix a test that always failed on --embedded --- .../suite/innodb/r/innodb-lru-force-no-free-page.result | 5 +++-- .../suite/innodb/t/innodb-lru-force-no-free-page.test | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result index d095f4e32e7..dba01945de3 100644 --- a/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result +++ b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result @@ -1,9 +1,10 @@ -call mtr.add_suppression("\\[Warning\\] InnoDB: Difficult to find free blocks in the buffer pool."); +call mtr.add_suppression("InnoDB: Difficult to find free blocks in the buffer pool"); +SET @saved_debug = @@SESSION.debug_dbug; SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; BEGIN; INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); COMMIT; -SET SESSION debug_dbug=""; +SET debug_dbug = @saved_debug; DROP TABLE t1; FOUND /InnoDB: Difficult to find free blocks / in mysqld.1.err diff --git a/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test index e358446eca9..fe04f4b7705 100644 --- a/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test +++ b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test @@ -1,8 +1,10 @@ --source include/have_innodb.inc --source include/have_debug.inc +--source include/not_embedded.inc -call mtr.add_suppression("\\[Warning\\] InnoDB: Difficult to find free blocks in the buffer pool."); +call mtr.add_suppression("InnoDB: Difficult to find free blocks in the buffer pool"); +SET @saved_debug = @@SESSION.debug_dbug; SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; @@ -10,7 +12,7 @@ BEGIN; INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); COMMIT; -SET SESSION debug_dbug=""; +SET debug_dbug = @saved_debug; DROP TABLE t1; @@ -21,4 +23,3 @@ let SEARCH_RANGE= -50000; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; --let SEARCH_PATTERN=InnoDB: Difficult to find free blocks --source include/search_pattern_in_file.inc - From 9c6fc7b644da912a0e68c720f2234a8554bbfeb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 16:38:16 +0200 Subject: [PATCH 54/82] Fix -Wsign-compare introduced by Compilation speed (#546) --- sql/key.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/key.cc b/sql/key.cc index af473d85847..0040647b5c7 100644 --- a/sql/key.cc +++ b/sql/key.cc @@ -1,4 +1,5 @@ /* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2018, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -132,7 +133,7 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, Don't copy data for null values The -1 below is to subtract the null byte which is already handled */ - length= MY_MIN(key_length, key_part->store_length-1); + length= MY_MIN(key_length, uint(key_part->store_length)-1); if (with_zerofill) bzero((char*) to_key, length); continue; @@ -202,7 +203,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, Don't copy data for null bytes The -1 below is to subtract the null byte which is already handled */ - length= MY_MIN(key_length, key_part->store_length-1); + length= MY_MIN(key_length, uint(key_part->store_length)-1); continue; } } From abbce9ed560d16bd76a84b30d4f4f731f0b56dc0 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 22 Dec 2017 15:03:24 +0200 Subject: [PATCH 55/82] Fixed compiler warnings in guess_malloc_library --- mysys/guess_malloc_library.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mysys/guess_malloc_library.c b/mysys/guess_malloc_library.c index 2e640757e11..70e3c7ca0f2 100644 --- a/mysys/guess_malloc_library.c +++ b/mysys/guess_malloc_library.c @@ -20,8 +20,13 @@ #include #include +typedef const char* (*tc_version_type)(int*, int*, const char**); +typedef int (*mallctl_type)(const char*, void*, size_t*, void*, size_t); + char *guess_malloc_library() { + tc_version_type tc_version_func; + mallctl_type mallctl_func; #ifndef HAVE_DLOPEN return (char*) MALLOC_LIBRARY; #else @@ -33,9 +38,7 @@ char *guess_malloc_library() } /* tcmalloc */ - typedef const char* (*tc_version_type)(int*, int*, const char**); - tc_version_type tc_version_func = - (tc_version_type) dlsym(RTLD_DEFAULT, "tc_version"); + tc_version_func= (tc_version_type) dlsym(RTLD_DEFAULT, "tc_version"); if (tc_version_func) { int major, minor; @@ -45,9 +48,7 @@ char *guess_malloc_library() } /* jemalloc */ - typedef int (*mallctl_type)(const char*, void*, size_t*, void*, size_t); - mallctl_type mallctl_func = - (mallctl_type) dlsym(RTLD_DEFAULT, "mallctl"); + mallctl_func= (mallctl_type) dlsym(RTLD_DEFAULT, "mallctl"); if (mallctl_func) { char *ver; From 3fdd390791ac91ecdc545fd1686a7ed43e8f1945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 19:02:38 +0200 Subject: [PATCH 56/82] MDEV-14441 InnoDB hangs when setting innodb_adaptive_hash_index=OFF during UPDATE This race condition is a regression caused by MDEV-12121. btr_cur_update_in_place(): Determine block->index!=NULL only once in order to determine whether an adaptive hash index bucket needs to be exclusively locked and unlocked. If we evaluated block->index multiple times, and the adaptive hash index was disabled before we locked the adaptive hash index, then we would never release the adaptive hash index bucket latch, which would eventually lead to InnoDB hanging. --- storage/innobase/btr/btr0cur.cc | 42 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 6acab881288..2830a38fa4a 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3,7 +3,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -3692,34 +3692,38 @@ btr_cur_update_in_place( || row_get_rec_trx_id(rec, index, offsets)); #ifdef BTR_CUR_HASH_ADAPT - if (block->index) { - /* TO DO: Can we skip this if none of the fields - index->search_info->curr_n_fields - are being updated? */ + { + rw_lock_t* ahi_latch = block->index + ? btr_get_search_latch(block->index) : NULL; + if (ahi_latch) { + /* TO DO: Can we skip this if none of the fields + index->search_info->curr_n_fields + are being updated? */ - /* The function row_upd_changes_ord_field_binary works only - if the update vector was built for a clustered index, we must - NOT call it if index is secondary */ + /* The function row_upd_changes_ord_field_binary + does not work on a secondary index. */ - if (!dict_index_is_clust(index) - || row_upd_changes_ord_field_binary(index, update, thr, - NULL, NULL)) { + if (!dict_index_is_clust(index) + || row_upd_changes_ord_field_binary( + index, update, thr, NULL, NULL)) { - /* Remove possible hash index pointer to this record */ - btr_search_update_hash_on_delete(cursor); + /* Remove possible hash index pointer + to this record */ + btr_search_update_hash_on_delete(cursor); + } } - btr_search_x_lock(index); - } + rw_lock_x_lock(ahi_latch); - assert_block_ahi_valid(block); + assert_block_ahi_valid(block); #endif /* BTR_CUR_HASH_ADAPT */ - row_upd_rec_in_place(rec, index, offsets, update, page_zip); + row_upd_rec_in_place(rec, index, offsets, update, page_zip); #ifdef BTR_CUR_HASH_ADAPT - if (block->index) { - btr_search_x_unlock(index); + if (ahi_latch) { + rw_lock_x_unlock(ahi_latch); + } } #endif /* BTR_CUR_HASH_ADAPT */ From 458e33cfbc53d5e3e6e2857688ae98efcef1faf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 17:06:26 +0200 Subject: [PATCH 57/82] MDEV-14441 Deadlock due to InnoDB adaptive hash index This is not fixing the reported problem, but a potential problem that was introduced in MDEV-11369. row_sel_try_search_shortcut(), row_sel_try_search_shortcut_for_mysql(): When an adaptive hash index search lands on top of rec_is_default_row(), we must skip the candidate and perform a normal search. This is because the adaptive hash index latch only protects the record from being deleted but does not prevent concurrent inserts into the page. Therefore, it is not safe to dereference the next-record pointer. --- storage/innobase/row/row0sel.cc | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 1395faaad96..083ffba33f8 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -2,7 +2,7 @@ Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1487,20 +1487,11 @@ row_sel_try_search_shortcut( rec = btr_pcur_get_rec(&(plan->pcur)); - if (!page_rec_is_user_rec(rec)) { + if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { return(SEL_RETRY); } - if (rec_is_default_row(rec, index)) { - /* Skip the 'default row' pseudo-record. */ - if (!btr_pcur_move_to_next_user_rec(&plan->pcur, mtr)) { - return(SEL_RETRY); - } - - rec = btr_pcur_get_rec(&plan->pcur); - } - ut_ad(plan->mode == PAGE_CUR_GE); /* As the cursor is now placed on a user record after a search with @@ -3908,20 +3899,11 @@ row_sel_try_search_shortcut_for_mysql( BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr); rec = btr_pcur_get_rec(pcur); - if (!page_rec_is_user_rec(rec)) { + if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { return(SEL_RETRY); } - if (rec_is_default_row(rec, index)) { - /* Skip the 'default row' pseudo-record. */ - if (!btr_pcur_move_to_next_user_rec(pcur, mtr)) { - return(SEL_RETRY); - } - - rec = btr_pcur_get_rec(pcur); - } - /* As the cursor is now placed on a user record after a search with the mode PAGE_CUR_GE, the up_match field in the cursor tells how many fields in the user record matched to the search tuple */ From 12f804acfa45dac5af04dd7da37670c50f4d8a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 18:08:59 +0200 Subject: [PATCH 58/82] MDEV-14441 Deadlock due to InnoDB adaptive hash index This is mere code clean-up; the reported problem was already fixed in commit 3fdd390791ac91ecdc545fd1686a7ed43e8f1945. row_sel(): Remove the variable search_latch_locked. row_sel_try_search_shortcut(): Remove the parameter search_latch_locked, which was always passed as nonzero. row_sel_try_search_shortcut(), row_sel_try_search_shortcut_for_mysql(): Do not expect the caller to acquire the AHI latch. Instead, acquire and release it inside this function. row_search_mvcc(): Remove a bogus condition on mysql_n_tables_locked. When the btr_search_latch was split into an array of latches in MySQL 5.7.8 as part of the Oracle Bug#20985298 fix, the "caching" of the latch across storage engine API calls was removed, and thus it is unnecessary to avoid adaptive hash index searches during INSERT...SELECT. --- storage/innobase/row/row0sel.cc | 148 ++++++++------------------------ 1 file changed, 36 insertions(+), 112 deletions(-) diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 083ffba33f8..c0d56bf0bfa 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -1462,33 +1462,24 @@ row_sel_try_search_shortcut( sel_node_t* node, /*!< in: select node for a consistent read */ plan_t* plan, /*!< in: plan for a unique search in clustered index */ - ibool search_latch_locked, - /*!< in: whether the search holds latch on - search system. */ mtr_t* mtr) /*!< in: mtr */ { - dict_index_t* index; - rec_t* rec; - mem_heap_t* heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - ulint ret; - rec_offs_init(offsets_); - - index = plan->index; + dict_index_t* index = plan->index; ut_ad(node->read_view); ut_ad(plan->unique_search); ut_ad(!plan->must_get_clust); - ut_ad(!search_latch_locked - || rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - row_sel_open_pcur(plan, search_latch_locked, mtr); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); - rec = btr_pcur_get_rec(&(plan->pcur)); + row_sel_open_pcur(plan, TRUE, mtr); + + const rec_t* rec = btr_pcur_get_rec(&(plan->pcur)); if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { - +retry: + rw_lock_s_unlock(ahi_latch); return(SEL_RETRY); } @@ -1499,36 +1490,34 @@ row_sel_try_search_shortcut( fields in the user record matched to the search tuple */ if (btr_pcur_get_up_match(&(plan->pcur)) < plan->n_exact_match) { - +exhausted: + rw_lock_s_unlock(ahi_latch); return(SEL_EXHAUSTED); } /* This is a non-locking consistent read: if necessary, fetch a previous version of the record */ + mem_heap_t* heap = NULL; + ulint offsets_[REC_OFFS_NORMAL_SIZE]; + ulint* offsets = offsets_; + rec_offs_init(offsets_); offsets = rec_get_offsets(rec, index, offsets, true, ULINT_UNDEFINED, &heap); if (dict_index_is_clust(index)) { if (!lock_clust_rec_cons_read_sees(rec, index, offsets, node->read_view)) { - ret = SEL_RETRY; - goto func_exit; + goto retry; } } else if (!srv_read_only_mode && !lock_sec_rec_cons_read_sees( rec, index, node->read_view)) { - - ret = SEL_RETRY; - goto func_exit; + goto retry; } - /* Test the deleted flag. */ - if (rec_get_deleted_flag(rec, dict_table_is_comp(plan->table))) { - - ret = SEL_EXHAUSTED; - goto func_exit; + goto exhausted; } /* Fetch the columns needed in test conditions. The index @@ -1542,20 +1531,18 @@ row_sel_try_search_shortcut( /* Test the rest of search conditions */ if (!row_sel_test_other_conds(plan)) { - - ret = SEL_EXHAUSTED; - goto func_exit; + goto exhausted; } ut_ad(plan->pcur.latch_mode == BTR_SEARCH_LEAF); plan->n_rows_fetched++; - ret = SEL_FOUND; -func_exit: + rw_lock_s_unlock(ahi_latch); + if (UNIV_LIKELY_NULL(heap)) { mem_heap_free(heap); } - return(ret); + return(SEL_FOUND); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -1602,12 +1589,6 @@ row_sel( ut_ad(thr->run_node == node); -#ifdef BTR_CUR_HASH_ADAPT - ibool search_latch_locked = FALSE; -#else /* BTR_CUR_HASH_ADAPT */ -# define search_latch_locked false -#endif /* BTR_CUR_HASH_ADAPT */ - if (node->read_view) { /* In consistent reads, we try to do with the hash index and not to use the buffer page get. This is to reduce memory bus @@ -1656,33 +1637,14 @@ table_loop: #ifdef BTR_CUR_HASH_ADAPT if (consistent_read && plan->unique_search && !plan->pcur_is_open && !plan->must_get_clust) { - if (!search_latch_locked) { - btr_search_s_lock(index); - - search_latch_locked = TRUE; - } else if (rw_lock_get_writer(btr_get_search_latch(index)) - == RW_LOCK_X_WAIT) { - - /* There is an x-latch request waiting: release the - s-latch for a moment; as an s-latch here is often - kept for some 10 searches before being released, - a waiting x-latch request would block other threads - from acquiring an s-latch for a long time, lowering - performance significantly in multiprocessors. */ - - btr_search_s_unlock(index); - btr_search_s_lock(index); - } - - switch (row_sel_try_search_shortcut(node, plan, - search_latch_locked, - &mtr)) { + switch (row_sel_try_search_shortcut(node, plan, &mtr)) { case SEL_FOUND: goto next_table; case SEL_EXHAUSTED: goto table_exhausted; default: ut_ad(0); + /* fall through */ case SEL_RETRY: break; } @@ -1692,19 +1654,13 @@ table_loop: mtr_commit(&mtr); mtr_start(&mtr); } - - if (search_latch_locked) { - btr_search_s_unlock(index); - - search_latch_locked = FALSE; - } #endif /* BTR_CUR_HASH_ADAPT */ if (!plan->pcur_is_open) { /* Evaluate the expressions to build the search tuple and open the cursor */ - row_sel_open_pcur(plan, search_latch_locked, &mtr); + row_sel_open_pcur(plan, FALSE, &mtr); cursor_just_opened = TRUE; @@ -2107,8 +2063,6 @@ skip_lock: } next_rec: - ut_ad(!search_latch_locked); - if (mtr_has_extra_clust_latch) { /* We must commit &mtr if we are moving to the next @@ -2146,8 +2100,6 @@ next_table: plan->cursor_at_end = TRUE; } else { - ut_ad(!search_latch_locked); - plan->stored_cursor_rec_processed = TRUE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2238,8 +2190,6 @@ stop_for_a_while: inserted new records which should have appeared in the result set, which would result in the phantom problem. */ - ut_ad(!search_latch_locked); - plan->stored_cursor_rec_processed = FALSE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2256,7 +2206,6 @@ commit_mtr_for_a_while: plan->stored_cursor_rec_processed = TRUE; - ut_ad(!search_latch_locked); btr_pcur_store_position(&(plan->pcur), &mtr); mtr_commit(&mtr); @@ -2270,7 +2219,6 @@ lock_wait_or_error: /* See the note at stop_for_a_while: the same holds for this case */ ut_ad(!btr_pcur_is_before_first_on_page(&plan->pcur) || !node->asc); - ut_ad(!search_latch_locked); plan->stored_cursor_rec_processed = FALSE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2278,11 +2226,6 @@ lock_wait_or_error: mtr_commit(&mtr); func_exit: -#ifdef BTR_CUR_HASH_ADAPT - if (search_latch_locked) { - btr_search_s_unlock(index); - } -#endif /* BTR_CUR_HASH_ADAPT */ ut_ad(!sync_check_iterate(dict_sync_check())); if (heap != NULL) { @@ -3895,12 +3838,15 @@ row_sel_try_search_shortcut_for_mysql( ut_ad(dict_index_is_clust(index)); ut_ad(!prebuilt->templ_contains_blob); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE, BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr); rec = btr_pcur_get_rec(pcur); if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { - +retry: + rw_lock_s_unlock(ahi_latch); return(SEL_RETRY); } @@ -3909,7 +3855,8 @@ row_sel_try_search_shortcut_for_mysql( fields in the user record matched to the search tuple */ if (btr_pcur_get_up_match(pcur) < dtuple_get_n_fields(search_tuple)) { - +exhausted: + rw_lock_s_unlock(ahi_latch); return(SEL_EXHAUSTED); } @@ -3921,20 +3868,19 @@ row_sel_try_search_shortcut_for_mysql( if (!lock_clust_rec_cons_read_sees( rec, index, *offsets, trx_get_read_view(trx))) { - - return(SEL_RETRY); + goto retry; } if (rec_get_deleted_flag(rec, dict_table_is_comp(index->table))) { /* In delete-marked records, DB_TRX_ID must always refer to an existing undo log record. */ ut_ad(row_get_rec_trx_id(rec, index, *offsets)); - - return(SEL_EXHAUSTED); + goto exhausted; } *out_rec = rec; + rw_lock_s_unlock(ahi_latch); return(SEL_FOUND); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -4323,24 +4269,14 @@ row_search_mvcc( mode = PAGE_CUR_GE; - if (trx->mysql_n_tables_locked == 0 - && prebuilt->select_lock_type == LOCK_NONE + if (prebuilt->select_lock_type == LOCK_NONE && trx->isolation_level > TRX_ISO_READ_UNCOMMITTED && MVCC::is_view_active(trx->read_view)) { /* This is a SELECT query done as a consistent read, and the read view has already been allocated: let us try a search shortcut through the hash - index. - NOTE that we must also test that - mysql_n_tables_locked == 0, because this might - also be INSERT INTO ... SELECT ... or - CREATE TABLE ... SELECT ... . Our algorithm is - NOT prepared to inserts interleaved with the SELECT, - and if we try that, we can deadlock on the adaptive - hash index semaphore! */ - - rw_lock_s_lock(btr_get_search_latch(index)); + index. */ switch (row_sel_try_search_shortcut_for_mysql( &rec, prebuilt, &offsets, &heap, @@ -4388,27 +4324,17 @@ row_search_mvcc( shortcut_match: mtr_commit(&mtr); - /* NOTE that we do NOT store the cursor position */ - err = DB_SUCCESS; - - rw_lock_s_unlock(btr_get_search_latch(index)); - goto func_exit; case SEL_EXHAUSTED: shortcut_mismatch: mtr_commit(&mtr); - - err = DB_RECORD_NOT_FOUND; - - rw_lock_s_unlock(btr_get_search_latch(index)); - /* NOTE that we do NOT store the cursor position */ - + err = DB_RECORD_NOT_FOUND; goto func_exit; case SEL_RETRY: @@ -4420,8 +4346,6 @@ row_search_mvcc( mtr_commit(&mtr); mtr_start(&mtr); - - rw_lock_s_unlock(btr_get_search_latch(index)); } } #endif /* BTR_CUR_HASH_ADAPT */ From 542ad0fa3f1e72a83190f8ba95c7ad616213fc9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 17:58:16 +0200 Subject: [PATCH 59/82] btr_search_check_guess(): Remove the parameter 'mode' Also, use 32-bit native reads to read the 32-bit aligned FIL_PAGE_PREV and FIL_PAGE_NEXT reads, to compare them to the byte order agnostic pattern FIL_NULL (0xffffffff). --- storage/innobase/btr/btr0sea.cc | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 6613e4c59f8..c154af9eccc 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -791,16 +791,14 @@ TRUE, then cursor->up_match and cursor->low_match both have sensible values. previous record to check our guess! @param[in] tuple data tuple @param[in] mode PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, PAGE_CUR_GE -@param[in] mtr mini transaction -@return TRUE if success */ +@return whether a match was found */ static -ibool +bool btr_search_check_guess( btr_cur_t* cursor, - ibool can_only_compare_to_cursor_rec, + bool can_only_compare_to_cursor_rec, const dtuple_t* tuple, - ulint mode, - mtr_t* mtr) + ulint mode) { rec_t* rec; ulint n_unique; @@ -862,14 +860,13 @@ btr_search_check_guess( match = 0; if ((mode == PAGE_CUR_G) || (mode == PAGE_CUR_GE)) { - rec_t* prev_rec; - ut_ad(!page_rec_is_infimum(rec)); - prev_rec = page_rec_get_prev(rec); + const rec_t* prev_rec = page_rec_get_prev(rec); if (page_rec_is_infimum(prev_rec)) { - success = btr_page_get_prev(page_align(prev_rec), mtr) + success = *reinterpret_cast( + page_align(prev_rec) + FIL_PAGE_PREV) == FIL_NULL; goto exit_func; @@ -884,17 +881,14 @@ btr_search_check_guess( } else { success = cmp >= 0; } - - goto exit_func; } else { - rec_t* next_rec; - ut_ad(!page_rec_is_supremum(rec)); - next_rec = page_rec_get_next(rec); + const rec_t* next_rec = page_rec_get_next(rec); if (page_rec_is_supremum(next_rec)) { - if (btr_page_get_next(page_align(next_rec), mtr) + if (*reinterpret_cast( + page_align(next_rec) + FIL_PAGE_NEXT) == FIL_NULL) { cursor->up_match = 0; @@ -1094,7 +1088,7 @@ btr_search_guess_on_hash( if (index_id != btr_page_get_index_id(block->frame) || !btr_search_check_guess(cursor, has_search_latch, - tuple, mode, mtr)) { + tuple, mode)) { if (!has_search_latch) { btr_leaf_page_release(block, latch_mode, mtr); From 4beb699a361cd28c605e94029330fd3701769f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 17:35:34 +0200 Subject: [PATCH 60/82] MDEV-14952 Avoid repeated calls to btr_get_search_latch() btr_cur_search_to_nth_level(), row_sel(): Do not bother to yield to waiting exclusive lock requests on the adaptive hash index latch. When the btr_search_latch was split into an array of latches in MySQL 5.7.8 as part of the Oracle Bug#20985298 fix, the "caching" of the latch across storage engine API calls was removed. Thus, X-lock requests should have a good chance of becoming served, and starvation should not be possible. btr_search_guess_on_hash(): Clean up a debug assertion. --- storage/innobase/btr/btr0cur.cc | 4 +--- storage/innobase/btr/btr0sea.cc | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 74940123d1a..b6b3cb041ea 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3,7 +3,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1129,8 +1129,6 @@ btr_cur_search_to_nth_level( && btr_search_enabled && !modify_external && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG) - && rw_lock_get_writer(btr_get_search_latch(index)) - == RW_LOCK_NOT_LOCKED && btr_search_guess_on_hash(index, info, tuple, mode, latch_mode, cursor, has_search_latch, mtr)) { diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index c154af9eccc..eeb43f513de 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1021,8 +1021,7 @@ btr_search_guess_on_hash( } } - ut_ad(rw_lock_get_writer(btr_get_search_latch(index)) != RW_LOCK_X); - ut_ad(rw_lock_get_reader_count(btr_get_search_latch(index)) > 0); + ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); rec = (rec_t*) ha_search_and_get_data( btr_get_search_table(index), fold); From 0664d633e415d04a90f343dbaf954ae756a474b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 19:51:09 +0200 Subject: [PATCH 61/82] MDEV-14952 Avoid repeated calls to btr_get_search_latch() btr_cur_search_to_nth_level(), btr_search_guess_on_hash(), btr_pcur_open_with_no_init_func(), row_sel_open_pcur(): Replace the parameter has_search_latch with the ahi_latch (passed as NULL if the caller does not hold the latch). btr_search_update_hash_node_on_insert(), btr_search_update_hash_on_insert(), btr_search_build_page_hash_index(): Add the parameter ahi_latch. btr_search_x_lock(), btr_search_x_unlock(), btr_search_s_lock(), btr_search_s_unlock(): Remove. --- storage/innobase/btr/btr0cur.cc | 53 +++-- storage/innobase/btr/btr0pcur.cc | 6 +- storage/innobase/btr/btr0sea.cc | 314 +++++++++++++-------------- storage/innobase/include/btr0cur.h | 29 ++- storage/innobase/include/btr0pcur.h | 27 ++- storage/innobase/include/btr0pcur.ic | 34 +-- storage/innobase/include/btr0sea.h | 55 ++--- storage/innobase/include/btr0sea.ic | 47 +--- storage/innobase/row/row0sel.cc | 30 ++- 9 files changed, 270 insertions(+), 325 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index b6b3cb041ea..cda0636146f 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -919,8 +919,7 @@ search tuple should be performed in the B-tree. InnoDB does an insert immediately after the cursor. Thus, the cursor may end up on a user record, or on a page infimum record. */ dberr_t -btr_cur_search_to_nth_level( -/*========================*/ +btr_cur_search_to_nth_level_func( dict_index_t* index, /*!< in: index */ ulint level, /*!< in: the tree level of search */ const dtuple_t* tuple, /*!< in: data tuple; NOTE: n_fields_cmp in @@ -935,17 +934,16 @@ btr_cur_search_to_nth_level( cursor->left_block is used to store a pointer to the left neighbor page, in the cases BTR_SEARCH_PREV and BTR_MODIFY_PREV; - NOTE that if has_search_latch - is != 0, we maybe do not have a latch set - on the cursor page, we assume - the caller uses his search latch - to protect the record! */ + NOTE that if ahi_latch, we might not have a + cursor page latch, we assume that ahi_latch + protects the record! */ btr_cur_t* cursor, /*!< in/out: tree cursor; the cursor page is s- or x-latched, but see also above! */ - ulint has_search_latch, - /*!< in: info on the latch mode the - caller currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: currently held btr_search_latch + (in RW_S_LATCH mode), or NULL */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr, /*!< in: mtr */ @@ -1123,7 +1121,7 @@ btr_cur_search_to_nth_level( && mode != PAGE_CUR_LE_OR_EXTENDS # endif /* PAGE_CUR_LE_OR_EXTENDS */ && !dict_index_is_spatial(index) - /* If !has_search_latch, we do a dirty read of + /* If !ahi_latch, we do a dirty read of btr_search_enabled below, and btr_search_guess_on_hash() will have to check it again. */ && btr_search_enabled @@ -1131,7 +1129,7 @@ btr_cur_search_to_nth_level( && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG) && btr_search_guess_on_hash(index, info, tuple, mode, latch_mode, cursor, - has_search_latch, mtr)) { + ahi_latch, mtr)) { /* Search using the hash index succeeded */ @@ -1152,10 +1150,12 @@ btr_cur_search_to_nth_level( /* If the hash search did not succeed, do binary search down the tree */ - if (has_search_latch) { +#ifdef BTR_CUR_HASH_ADAPT + if (ahi_latch) { /* Release possible search latch to obey latching order */ - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); } +#endif /* BTR_CUR_HASH_ADAPT */ /* Store the position of the tree latch we push to mtr so that we know how to release it when we have latched leaf node(s) */ @@ -2222,15 +2222,17 @@ func_exit: ut_free(prev_tree_savepoints); } - if (has_search_latch) { - btr_search_s_lock(index); - } - if (mbr_adj) { /* remember that we will need to adjust parent MBR */ cursor->rtr_info->mbr_adj = true; } +#ifdef BTR_CUR_HASH_ADAPT + if (ahi_latch) { + rw_lock_s_lock(ahi_latch); + } +#endif /* BTR_CUR_HASH_ADAPT */ + DBUG_RETURN(err); } @@ -3300,10 +3302,14 @@ fail_err: ut_ad(entry->info_bits == REC_INFO_DEFAULT_ROW); ut_ad(index->is_instant()); ut_ad(flags == BTR_NO_LOCKING_FLAG); - } else if (!reorg && cursor->flag == BTR_CUR_HASH) { - btr_search_update_hash_node_on_insert(cursor); } else { - btr_search_update_hash_on_insert(cursor); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + if (!reorg && cursor->flag == BTR_CUR_HASH) { + btr_search_update_hash_node_on_insert( + cursor, ahi_latch); + } else { + btr_search_update_hash_on_insert(cursor, ahi_latch); + } } #endif /* BTR_CUR_HASH_ADAPT */ @@ -3509,7 +3515,8 @@ btr_cur_pessimistic_insert( ut_ad((flags & ~BTR_KEEP_IBUF_BITMAP) == BTR_NO_LOCKING_FLAG); } else { - btr_search_update_hash_on_insert(cursor); + btr_search_update_hash_on_insert( + cursor, btr_get_search_latch(index)); } #endif /* BTR_CUR_HASH_ADAPT */ if (inherit && !(flags & BTR_NO_LOCKING_FLAG)) { diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 2b85c764a3b..56f6da81ded 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -353,7 +353,11 @@ btr_pcur_restore_position_func( } btr_pcur_open_with_no_init_func(index, tuple, mode, latch_mode, - cursor, 0, file, line, mtr); + cursor, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr); /* Restore the old search mode */ cursor->search_mode = old_mode; diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index eeb43f513de..1a70dce1c7b 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -48,7 +48,7 @@ Search system is protected by array of latches. */ char btr_search_enabled = true; /** Number of adaptive hash index partition. */ -ulong btr_ahi_parts = 8; +ulong btr_ahi_parts; #ifdef UNIV_SEARCH_PERF_STAT /** Number of successful adaptive hash index lookups */ @@ -177,23 +177,6 @@ btr_search_get_n_fields( return(btr_search_get_n_fields(cursor->n_fields, cursor->n_bytes)); } -/********************************************************************//** -Builds a hash index on a page with the given parameters. If the page already -has a hash index with different parameters, the old hash index is removed. -If index is non-NULL, this function checks if n_fields and n_bytes are -sensible values, and does not build a hash index if not. */ -static -void -btr_search_build_page_hash_index( -/*=============================*/ - dict_index_t* index, /*!< in: index for which to build, or NULL if - not known */ - buf_block_t* block, /*!< in: index page, s- or x-latched */ - ulint n_fields,/*!< in: hash this many full fields */ - ulint n_bytes,/*!< in: hash this many bytes from the next - field */ - ibool left_side);/*!< in: hash for searches from left side? */ - /** This function should be called before reserving any btr search mutex, if the intended operation might add nodes to the search system hash table. Because of the latching order, once we have reserved the btr search system @@ -224,8 +207,9 @@ btr_search_check_free_space_in_heap(dict_index_t* index) if (heap->free_block == NULL) { buf_block_t* block = buf_block_alloc(NULL); + rw_lock_t* ahi_latch = btr_get_search_latch(index); - btr_search_x_lock(index); + rw_lock_x_lock(ahi_latch); if (btr_search_enabled && heap->free_block == NULL) { @@ -234,7 +218,7 @@ btr_search_check_free_space_in_heap(dict_index_t* index) buf_block_free(block); } - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } } @@ -457,12 +441,10 @@ btr_search_info_get_ref_count( ut_ad(info); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); - - btr_search_s_lock(index); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); ret = info->ref_count; - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); return(ret); } @@ -723,61 +705,6 @@ btr_search_update_hash_ref( } } -/** Updates the search info. -@param[in,out] info search info -@param[in] cursor cursor which was just positioned */ -void -btr_search_info_update_slow( - btr_search_t* info, - btr_cur_t* cursor) -{ - buf_block_t* block; - ibool build_index; - - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); - - block = btr_cur_get_block(cursor); - - /* NOTE that the following two function calls do NOT protect - info or block->n_fields etc. with any semaphore, to save CPU time! - We cannot assume the fields are consistent when we return from - those functions! */ - - btr_search_info_update_hash(info, cursor); - - build_index = btr_search_update_block_hash_info(info, block, cursor); - - if (build_index || (cursor->flag == BTR_CUR_HASH_FAIL)) { - - btr_search_check_free_space_in_heap(cursor->index); - } - - if (cursor->flag == BTR_CUR_HASH_FAIL) { - /* Update the hash node reference, if appropriate */ - -#ifdef UNIV_SEARCH_PERF_STAT - btr_search_n_hash_fail++; -#endif /* UNIV_SEARCH_PERF_STAT */ - - btr_search_x_lock(cursor->index); - - btr_search_update_hash_ref(info, block, cursor); - - btr_search_x_unlock(cursor->index); - } - - if (build_index) { - /* Note that since we did not protect block->n_fields etc. - with any semaphore, the values can be inconsistent. We have - to check inside the function call that they make sense. */ - btr_search_build_page_hash_index(cursor->index, block, - block->n_fields, - block->n_bytes, - block->left_side); - } -} - /** Checks if a guessed position for a tree cursor is right. Note that if mode is PAGE_CUR_LE, which is used in inserts, and the function returns TRUE, then cursor->up_match and cursor->low_match both have sensible values. @@ -947,12 +874,11 @@ both have sensible values. we assume the caller uses his search latch to protect the record! @param[out] cursor tree cursor -@param[in] has_search_latch - latch mode the caller currently has on - search system: RW_S/X_LATCH or 0 +@param[in] ahi_latch the adaptive hash index latch being held, + or NULL @param[in] mtr mini transaction -@return TRUE if succeeded */ -ibool +@return whether the search succeeded */ +bool btr_search_guess_on_hash( dict_index_t* index, btr_search_t* info, @@ -960,7 +886,7 @@ btr_search_guess_on_hash( ulint mode, ulint latch_mode, btr_cur_t* cursor, - ulint has_search_latch, + rw_lock_t* ahi_latch, mtr_t* mtr) { const rec_t* rec; @@ -970,6 +896,8 @@ btr_search_guess_on_hash( btr_cur_t cursor2; btr_pcur_t pcur; #endif + ut_ad(!ahi_latch || rw_lock_own(ahi_latch, RW_LOCK_S) + || rw_lock_own(ahi_latch, RW_LOCK_X)); if (!btr_search_enabled) { return(FALSE); @@ -977,6 +905,7 @@ btr_search_guess_on_hash( ut_ad(index && info && tuple && cursor && mtr); ut_ad(!dict_index_is_ibuf(index)); + ut_ad(!ahi_latch || ahi_latch == btr_get_search_latch(index)); ut_ad((latch_mode == BTR_SEARCH_LEAF) || (latch_mode == BTR_MODIFY_LEAF)); @@ -1009,27 +938,27 @@ btr_search_guess_on_hash( cursor->fold = fold; cursor->flag = BTR_CUR_HASH; - if (!has_search_latch) { - btr_search_s_lock(index); + rw_lock_t* use_latch = ahi_latch ? NULL : btr_get_search_latch(index); + + if (use_latch) { + rw_lock_s_lock(use_latch); if (!btr_search_enabled) { - btr_search_s_unlock(index); - - btr_search_failure(info, cursor); - - return(FALSE); + goto fail; } + } else { + ut_ad(btr_search_enabled); } ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); rec = (rec_t*) ha_search_and_get_data( - btr_get_search_table(index), fold); + btr_get_search_table(index), fold); if (rec == NULL) { - - if (!has_search_latch) { - btr_search_s_unlock(index); + if (use_latch) { +fail: + rw_lock_s_unlock(use_latch); } btr_search_failure(info, cursor); @@ -1039,22 +968,15 @@ btr_search_guess_on_hash( buf_block_t* block = buf_block_from_ahi(rec); - if (!has_search_latch) { + if (use_latch) { if (!buf_page_get_known_nowait( latch_mode, block, BUF_MAKE_YOUNG, __FILE__, __LINE__, mtr)) { - - if (!has_search_latch) { - btr_search_s_unlock(index); - } - - btr_search_failure(info, cursor); - - return(FALSE); + goto fail; } - btr_search_s_unlock(index); + rw_lock_s_unlock(use_latch); buf_block_dbg_add_level(block, SYNC_TREE_NODE_FROM_HASH); } @@ -1063,7 +985,7 @@ btr_search_guess_on_hash( ut_ad(buf_block_get_state(block) == BUF_BLOCK_REMOVE_HASH); - if (!has_search_latch) { + if (!ahi_latch) { btr_leaf_page_release(block, latch_mode, mtr); } @@ -1085,11 +1007,9 @@ btr_search_guess_on_hash( record to determine if our guess for the cursor position is right. */ if (index_id != btr_page_get_index_id(block->frame) - || !btr_search_check_guess(cursor, - has_search_latch, - tuple, mode)) { + || !btr_search_check_guess(cursor, !!ahi_latch, tuple, mode)) { - if (!has_search_latch) { + if (!ahi_latch) { btr_leaf_page_release(block, latch_mode, mtr); } @@ -1110,7 +1030,7 @@ btr_search_guess_on_hash( info->last_hash_succ = FALSE; /* Currently, does not work if the following fails: */ - ut_ad(!has_search_latch); + ut_ad(!ahi_latch); btr_leaf_page_release(block, latch_mode, mtr); @@ -1143,7 +1063,7 @@ btr_search_guess_on_hash( #ifdef UNIV_SEARCH_PERF_STAT btr_search_n_succ++; #endif - if (!has_search_latch && buf_page_peek_if_too_old(&block->page)) { + if (!ahi_latch && buf_page_peek_if_too_old(&block->page)) { buf_page_make_young(&block->page); } @@ -1191,6 +1111,8 @@ retry: /* This debug check uses a dirty read that could theoretically cause false positives while buf_pool_clear_hash_index() is executing. */ assert_block_ahi_valid(block); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (index == NULL) { return; @@ -1214,9 +1136,6 @@ retry: % btr_ahi_parts; latch = btr_search_latches[ahi_slot]; - ut_ad(!btr_search_own_any(RW_LOCK_S)); - ut_ad(!btr_search_own_any(RW_LOCK_X)); - rw_lock_s_lock(latch); assert_block_ahi_valid(block); @@ -1419,6 +1338,7 @@ If index is non-NULL, this function checks if n_fields and n_bytes are sensible, and does not build a hash index if not. @param[in,out] index index for which to build. @param[in,out] block index page, s-/x- latched. +@param[in,out] ahi_latch the adaptive search latch @param[in] n_fields hash this many full fields @param[in] n_bytes hash this many bytes of the next field @param[in] left_side hash for searches from left side */ @@ -1427,12 +1347,11 @@ void btr_search_build_page_hash_index( dict_index_t* index, buf_block_t* block, + rw_lock_t* ahi_latch, ulint n_fields, ulint n_bytes, ibool left_side) { - hash_table_t* table; - page_t* page; const rec_t* rec; const rec_t* next_rec; ulint fold; @@ -1454,6 +1373,7 @@ btr_search_build_page_hash_index( } rec_offs_init(offsets_); + ut_ad(ahi_latch == btr_get_search_latch(index)); ut_ad(index); ut_ad(block->page.id.space() == index->space); ut_a(!dict_index_is_ibuf(index)); @@ -1463,20 +1383,17 @@ btr_search_build_page_hash_index( ut_ad(rw_lock_own(&(block->lock), RW_LOCK_S) || rw_lock_own(&(block->lock), RW_LOCK_X)); - btr_search_s_lock(index); + rw_lock_s_lock(ahi_latch); - table = btr_get_search_table(index); - page = buf_block_get_frame(block); + const bool rebuild = block->index + && (block->curr_n_fields != n_fields + || block->curr_n_bytes != n_bytes + || block->curr_left_side != left_side); - if (block->index && ((block->curr_n_fields != n_fields) - || (block->curr_n_bytes != n_bytes) - || (block->curr_left_side != left_side))) { - - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); + if (rebuild) { btr_search_drop_page_hash_index(block); - } else { - btr_search_s_unlock(index); } /* Check that the values for hash index build are sensible */ @@ -1491,6 +1408,7 @@ btr_search_build_page_hash_index( return; } + page_t* page = buf_block_get_frame(block); n_recs = page_get_n_recs(page); if (n_recs == 0) { @@ -1573,7 +1491,8 @@ btr_search_build_page_hash_index( btr_search_check_free_space_in_heap(index); - btr_search_x_lock(index); + hash_table_t* table = btr_get_search_table(index); + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto exit_func; @@ -1611,7 +1530,7 @@ btr_search_build_page_hash_index( MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_ADDED, n_cached); exit_func: assert_block_ahi_valid(block); - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); ut_free(folds); ut_free(recs); @@ -1620,6 +1539,60 @@ exit_func: } } +/** Updates the search info. +@param[in,out] info search info +@param[in,out] cursor cursor which was just positioned */ +void +btr_search_info_update_slow(btr_search_t* info, btr_cur_t* cursor) +{ + rw_lock_t* ahi_latch = btr_get_search_latch(cursor->index); + + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_S)); + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_X)); + + buf_block_t* block = btr_cur_get_block(cursor); + + /* NOTE that the following two function calls do NOT protect + info or block->n_fields etc. with any semaphore, to save CPU time! + We cannot assume the fields are consistent when we return from + those functions! */ + + btr_search_info_update_hash(info, cursor); + + bool build_index = btr_search_update_block_hash_info( + info, block, cursor); + + if (build_index || (cursor->flag == BTR_CUR_HASH_FAIL)) { + + btr_search_check_free_space_in_heap(cursor->index); + } + + if (cursor->flag == BTR_CUR_HASH_FAIL) { + /* Update the hash node reference, if appropriate */ + +#ifdef UNIV_SEARCH_PERF_STAT + btr_search_n_hash_fail++; +#endif /* UNIV_SEARCH_PERF_STAT */ + + rw_lock_x_lock(ahi_latch); + + btr_search_update_hash_ref(info, block, cursor); + + rw_lock_x_unlock(ahi_latch); + } + + if (build_index) { + /* Note that since we did not protect block->n_fields etc. + with any semaphore, the values can be inconsistent. We have + to check inside the function call that they make sense. */ + btr_search_build_page_hash_index(cursor->index, block, + ahi_latch, + block->n_fields, + block->n_bytes, + block->left_side); + } +} + /** Move or delete hash entries for moved records, usually in a page split. If new_block is already hashed, then any hash index for block is dropped. If new_block is not hashed, and block is hashed, then a new hash index is @@ -1638,19 +1611,27 @@ btr_search_move_or_delete_hash_entries( return; } + dict_index_t* index = block->index; + if (!index) { + index = new_block->index; + } else { + ut_ad(!new_block->index || index == new_block->index); + } assert_block_ahi_valid(block); assert_block_ahi_valid(new_block); + rw_lock_t* ahi_latch = index ? btr_get_search_latch(index) : NULL; + if (new_block->index) { btr_search_drop_page_hash_index(block); return; } - dict_index_t* index = block->index; if (!index) { return; } - btr_search_s_lock(index); + + rw_lock_s_lock(ahi_latch); if (block->index) { ulint n_fields = block->curr_n_fields; @@ -1661,19 +1642,20 @@ btr_search_move_or_delete_hash_entries( new_block->n_bytes = block->curr_n_bytes; new_block->left_side = left_side; - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); ut_a(n_fields > 0 || n_bytes > 0); btr_search_build_page_hash_index( - index, new_block, n_fields, n_bytes, left_side); + index, new_block, ahi_latch, + n_fields, n_bytes, left_side); ut_ad(n_fields == block->curr_n_fields); ut_ad(n_bytes == block->curr_n_bytes); ut_ad(left_side == block->curr_left_side); return; } - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); } /** Updates the page hash index when a single record is deleted from a page. @@ -1728,7 +1710,9 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) mem_heap_free(heap); } - btr_search_x_lock(index); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + + rw_lock_x_lock(ahi_latch); assert_block_ahi_valid(block); if (block->index) { @@ -1744,21 +1728,25 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) assert_block_ahi_valid(block); } - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } /** Updates the page hash index when a single record is inserted on a page. @param[in] cursor cursor which was positioned to the place to insert using btr_cur_search_, and the new record has been - inserted next to the cursor. */ + inserted next to the cursor. +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_node_on_insert(btr_cur_t* cursor) +btr_search_update_hash_node_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch) { hash_table_t* table; buf_block_t* block; dict_index_t* index; rec_t* rec; + ut_ad(ahi_latch == btr_get_search_latch(cursor->index)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); #ifdef MYSQL_INDEX_DISABLE_AHI if (cursor->index->disable_ahi) return; #endif @@ -1781,8 +1769,7 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) ut_a(cursor->index == index); ut_a(!dict_index_is_ibuf(index)); - - btr_search_x_lock(index); + rw_lock_x_lock(ahi_latch); if (!block->index) { @@ -1806,11 +1793,11 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) func_exit: assert_block_ahi_valid(block); - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } else { - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); - btr_search_update_hash_on_insert(cursor); + btr_search_update_hash_on_insert(cursor, ahi_latch); } } @@ -1818,9 +1805,10 @@ func_exit: @param[in,out] cursor cursor which was positioned to the place to insert using btr_cur_search_..., and the new record has been inserted next - to the cursor */ + to the cursor +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_on_insert(btr_cur_t* cursor) +btr_search_update_hash_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch) { hash_table_t* table; buf_block_t* block; @@ -1834,13 +1822,16 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) ulint n_fields; ulint n_bytes; ibool left_side; - ibool locked = FALSE; + bool locked = false; mem_heap_t* heap = NULL; ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; rec_offs_init(offsets_); + ut_ad(ahi_latch == btr_get_search_latch(cursor->index)); ut_ad(page_is_leaf(btr_cur_get_page(cursor))); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); #ifdef MYSQL_INDEX_DISABLE_AHI if (cursor->index->disable_ahi) return; #endif @@ -1899,10 +1890,8 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) fold = rec_fold(rec, offsets, n_fields, n_bytes, index->id); } else { if (left_side) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1917,10 +1906,8 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) if (fold != ins_fold) { if (!locked) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1938,11 +1925,9 @@ check_next_rec: if (page_rec_is_supremum(next_rec)) { if (!left_side) { - if (!locked) { - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1958,10 +1943,8 @@ check_next_rec: if (ins_fold != next_fold) { if (!locked) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1980,8 +1963,9 @@ function_exit: mem_heap_free(heap); } if (locked) { - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_X)); } #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 76f13325e2a..8d8fe0bc236 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -177,8 +177,7 @@ Note that if mode is PAGE_CUR_LE, which is used in inserts, then cursor->up_match and cursor->low_match both will have sensible values. If mode is PAGE_CUR_GE, then up_match will a have a sensible value. */ dberr_t -btr_cur_search_to_nth_level( -/*========================*/ +btr_cur_search_to_nth_level_func( dict_index_t* index, /*!< in: index */ ulint level, /*!< in: the tree level of search */ const dtuple_t* tuple, /*!< in: data tuple; NOTE: n_fields_cmp in @@ -197,23 +196,29 @@ btr_cur_search_to_nth_level( cursor->left_block is used to store a pointer to the left neighbor page, in the cases BTR_SEARCH_PREV and BTR_MODIFY_PREV; - NOTE that if has_search_latch - is != 0, we maybe do not have a latch set - on the cursor page, we assume - the caller uses his search latch - to protect the record! */ + NOTE that if ahi_latch, we might not have a + cursor page latch, we assume that ahi_latch + protects the record! */ btr_cur_t* cursor, /*!< in/out: tree cursor; the cursor page is s- or x-latched, but see also above! */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: currently held btr_search_latch + (in RW_S_LATCH mode), or NULL */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr, /*!< in/out: mini-transaction */ ib_uint64_t autoinc = 0); /*!< in: PAGE_ROOT_AUTO_INC to be written (0 if none) */ +#ifdef BTR_CUR_HASH_ADAPT +# define btr_cur_search_to_nth_level(i,l,t,m,lm,c,a,fi,li,mtr) \ + btr_cur_search_to_nth_level_func(i,l,t,m,lm,c,a,fi,li,mtr) +#else /* BTR_CUR_HASH_ADAPT */ +# define btr_cur_search_to_nth_level(i,l,t,m,lm,c,a,fi,li,mtr) \ + btr_cur_search_to_nth_level_func(i,l,t,m,lm,c,fi,li,mtr) +#endif /* BTR_CUR_HASH_ADAPT */ /*****************************************************************//** Opens a cursor at either end of an index. diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index fab934ca0ee..1d8690a3c90 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -136,20 +136,25 @@ btr_pcur_open_with_no_init_func( may end up on the previous page of the record! */ ulint latch_mode,/*!< in: BTR_SEARCH_LEAF, ...; - NOTE that if has_search_latch != 0 then - we maybe do not acquire a latch on the cursor - page, but assume that the caller uses his - btr search latch to protect the record! */ + NOTE that if ahi_latch then we might not + acquire a cursor page latch, but assume + that the ahi_latch protects the record! */ btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: adaptive hash index latch held + by the caller, or NULL if none */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr); /*!< in: mtr */ -#define btr_pcur_open_with_no_init(ix,t,md,l,cur,has,m) \ - btr_pcur_open_with_no_init_func(ix,t,md,l,cur,has,__FILE__,__LINE__,m) +#ifdef BTR_CUR_HASH_ADAPT +# define btr_pcur_open_with_no_init(ix,t,md,l,cur,ahi,m) \ + btr_pcur_open_with_no_init_func(ix,t,md,l,cur,ahi,__FILE__,__LINE__,m) +#else /* BTR_CUR_HASH_ADAPT */ +# define btr_pcur_open_with_no_init(ix,t,md,l,cur,ahi,m) \ + btr_pcur_open_with_no_init_func(ix,t,md,l,cur,__FILE__,__LINE__,m) +#endif /* BTR_CUR_HASH_ADAPT */ /*****************************************************************//** Opens a persistent cursor at either end of an index. */ diff --git a/storage/innobase/include/btr0pcur.ic b/storage/innobase/include/btr0pcur.ic index 4490942a2bb..cbe1d76b094 100644 --- a/storage/innobase/include/btr0pcur.ic +++ b/storage/innobase/include/btr0pcur.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -454,9 +454,12 @@ btr_pcur_open_low( ut_ad(!dict_index_is_spatial(index)); - err = btr_cur_search_to_nth_level( - index, level, tuple, mode, latch_mode, - btr_cursor, 0, file, line, mtr, autoinc); + err = btr_cur_search_to_nth_level_func( + index, level, tuple, mode, latch_mode, btr_cursor, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr, autoinc); if (err != DB_SUCCESS) { ib::warn() << " Error code: " << err @@ -491,15 +494,15 @@ btr_pcur_open_with_no_init_func( may end up on the previous page of the record! */ ulint latch_mode,/*!< in: BTR_SEARCH_LEAF, ...; - NOTE that if has_search_latch != 0 then - we maybe do not acquire a latch on the cursor - page, but assume that the caller uses his - btr search latch to protect the record! */ + NOTE that if ahi_latch then we might not + acquire a cursor page latch, but assume + that the ahi_latch protects the record! */ btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: adaptive hash index latch held + by the caller, or NULL if none */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr) /*!< in: mtr */ @@ -514,9 +517,12 @@ btr_pcur_open_with_no_init_func( btr_cursor = btr_pcur_get_btr_cur(cursor); - err = btr_cur_search_to_nth_level( + err = btr_cur_search_to_nth_level_func( index, 0, tuple, mode, latch_mode, btr_cursor, - has_search_latch, file, line, mtr); +#ifdef BTR_CUR_HASH_ADAPT + ahi_latch, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr); cursor->pos_state = BTR_PCUR_IS_POSITIONED; diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index bd1a72fc3ac..cddf846438d 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -91,12 +91,11 @@ both have sensible values. we assume the caller uses his search latch to protect the record! @param[out] cursor tree cursor -@param[in] has_search_latch - latch mode the caller currently has on - search system: RW_S/X_LATCH or 0 +@param[in] ahi_latch the adaptive hash index latch being held, + or NULL @param[in] mtr mini transaction -@return TRUE if succeeded */ -ibool +@return whether the search succeeded */ +bool btr_search_guess_on_hash( dict_index_t* index, btr_search_t* info, @@ -104,7 +103,7 @@ btr_search_guess_on_hash( ulint mode, ulint latch_mode, btr_cur_t* cursor, - ulint has_search_latch, + rw_lock_t* ahi_latch, mtr_t* mtr); /** Move or delete hash entries for moved records, usually in a page split. @@ -140,17 +139,19 @@ btr_search_drop_page_hash_when_freed( /** Updates the page hash index when a single record is inserted on a page. @param[in] cursor cursor which was positioned to the place to insert using btr_cur_search_, and the new record has been - inserted next to the cursor. */ + inserted next to the cursor. +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_node_on_insert(btr_cur_t* cursor); +btr_search_update_hash_node_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch); /** Updates the page hash index when a single record is inserted on a page. -@param[in] cursor cursor which was positioned to the +@param[in,out] cursor cursor which was positioned to the place to insert using btr_cur_search_..., and the new record has been inserted next - to the cursor */ + to the cursor +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_on_insert(btr_cur_t* cursor); +btr_search_update_hash_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch); /** Updates the page hash index when a single record is deleted from a page. @param[in] cursor cursor which was positioned on the record to delete @@ -163,18 +164,6 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor); bool btr_search_validate(); -/** X-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_lock(const dict_index_t* index); - -/** X-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_unlock(const dict_index_t* index); - /** Lock all search latches in exclusive mode. */ UNIV_INLINE void @@ -185,18 +174,6 @@ UNIV_INLINE void btr_search_x_unlock_all(); -/** S-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_lock(const dict_index_t* index); - -/** S-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_unlock(const dict_index_t* index); - /** Lock all search latches in shared mode. */ UNIV_INLINE void @@ -243,15 +220,11 @@ btr_get_search_table(const dict_index_t* index); #else /* BTR_CUR_HASH_ADAPT */ # define btr_search_sys_create(size) # define btr_search_drop_page_hash_index(block) -# define btr_search_s_lock(index) -# define btr_search_s_unlock(index) # define btr_search_s_lock_all(index) # define btr_search_s_unlock_all(index) -# define btr_search_x_lock(index) -# define btr_search_x_unlock(index) # define btr_search_info_update(index, cursor) # define btr_search_move_or_delete_hash_entries(new_block, block) -# define btr_search_update_hash_on_insert(cursor) +# define btr_search_update_hash_on_insert(cursor, ahi_latch) # define btr_search_update_hash_on_delete(cursor) # define btr_search_sys_resize(hash_size) #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/include/btr0sea.ic b/storage/innobase/include/btr0sea.ic index b5a7536a2b4..65b953961df 100644 --- a/storage/innobase/include/btr0sea.ic +++ b/storage/innobase/include/btr0sea.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -45,13 +46,11 @@ btr_search_info_create(mem_heap_t* heap) } #ifdef BTR_CUR_HASH_ADAPT -/*********************************************************************//** -Updates the search info. */ +/** Updates the search info. +@param[in,out] info search info +@param[in,out] cursor cursor which was just positioned */ void -btr_search_info_update_slow( -/*========================*/ - btr_search_t* info, /*!< in/out: search info */ - btr_cur_t* cursor);/*!< in: cursor which was just positioned */ +btr_search_info_update_slow(btr_search_t* info, btr_cur_t* cursor); /*********************************************************************//** Updates the search info. */ @@ -87,24 +86,6 @@ btr_search_info_update( btr_search_info_update_slow(info, cursor); } -/** X-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_lock(const dict_index_t* index) -{ - rw_lock_x_lock(btr_get_search_latch(index)); -} - -/** X-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_unlock(const dict_index_t* index) -{ - rw_lock_x_unlock(btr_get_search_latch(index)); -} - /** Lock all search latches in exclusive mode. */ UNIV_INLINE void @@ -125,24 +106,6 @@ btr_search_x_unlock_all() } } -/** S-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_lock(const dict_index_t* index) -{ - rw_lock_s_lock(btr_get_search_latch(index)); -} - -/** S-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_unlock(const dict_index_t* index) -{ - rw_lock_s_unlock(btr_get_search_latch(index)); -} - /** Lock all search latches in shared mode. */ UNIV_INLINE void diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index c0d56bf0bfa..d01b870fc09 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -1272,24 +1272,19 @@ static void row_sel_open_pcur( /*==============*/ - plan_t* plan, /*!< in: table plan */ - ibool search_latch_locked, - /*!< in: TRUE if the thread currently - has the search latch locked in - s-mode */ - mtr_t* mtr) /*!< in: mtr */ + plan_t* plan, /*!< in: table plan */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: the adaptive hash index latch */ +#endif /* BTR_CUR_HASH_ADAPT */ + mtr_t* mtr) /*!< in/out: mini-transaction */ { dict_index_t* index; func_node_t* cond; que_node_t* exp; ulint n_fields; - ulint has_search_latch = 0; /* RW_S_LATCH or 0 */ ulint i; - if (search_latch_locked) { - has_search_latch = RW_S_LATCH; - } - index = plan->index; /* Calculate the value of the search tuple: the exact match columns @@ -1325,7 +1320,7 @@ row_sel_open_pcur( btr_pcur_open_with_no_init(index, plan->tuple, plan->mode, BTR_SEARCH_LEAF, &plan->pcur, - has_search_latch, mtr); + ahi_latch, mtr); } else { /* Open the cursor to the start or the end of the index (FALSE: no init) */ @@ -1473,7 +1468,7 @@ row_sel_try_search_shortcut( rw_lock_t* ahi_latch = btr_get_search_latch(index); rw_lock_s_lock(ahi_latch); - row_sel_open_pcur(plan, TRUE, mtr); + row_sel_open_pcur(plan, ahi_latch, mtr); const rec_t* rec = btr_pcur_get_rec(&(plan->pcur)); @@ -1659,8 +1654,11 @@ table_loop: if (!plan->pcur_is_open) { /* Evaluate the expressions to build the search tuple and open the cursor */ - - row_sel_open_pcur(plan, FALSE, &mtr); + row_sel_open_pcur(plan, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + &mtr); cursor_just_opened = TRUE; @@ -3841,7 +3839,7 @@ row_sel_try_search_shortcut_for_mysql( rw_lock_t* ahi_latch = btr_get_search_latch(index); rw_lock_s_lock(ahi_latch); btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE, - BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr); + BTR_SEARCH_LEAF, pcur, ahi_latch, mtr); rec = btr_pcur_get_rec(pcur); if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { From da27dc995ee407cb56761393bc5dcdd48456d851 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 16 Jan 2018 09:18:22 +0000 Subject: [PATCH 62/82] Fix Windows build --- sql/rpl_gtid.cc | 2 +- sql/set_var.cc | 2 +- sql/sql_class.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 9f48f908102..78e6165a551 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -582,7 +582,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, bool table_opened= false; TABLE *table; list_element *delete_list= 0, *next, *cur, **next_ptr_ptr, **best_ptr_ptr; - uint64_t best_sub_id; + uint64 best_sub_id; element *elem; ulonglong thd_saved_option= thd->variables.option_bits; Query_tables_list lex_backup; diff --git a/sql/set_var.cc b/sql/set_var.cc index 311b33bc0dd..097124a0b3a 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -1323,7 +1323,7 @@ resolve_engine_list_item(THD *thd, plugin_ref *list, uint32 *idx, { LEX_CSTRING item_str; plugin_ref ref; - uint32_t i; + uint32 i; THD *thd_or_null = (temp_copy ? thd : NULL); item_str.str= pos; diff --git a/sql/sql_class.h b/sql/sql_class.h index 8747d658aea..bb957e9ea20 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -833,7 +833,7 @@ typedef struct system_status_var ulonglong table_open_cache_overflows; double last_query_cost; double cpu_time, busy_time; - uint32_t threads_running; + uint32 threads_running; /* Don't initialize */ /* Memory used for thread local storage */ int64 max_local_memory_used; From e260c6a087484973625445787d05e8b0f7ffe5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 12:04:01 +0000 Subject: [PATCH 63/82] Fix commit 9631d933fbc2ec99874 so Travis CI doesn't fail TokuDB and Mroonga are not built on Travis CI to save time. Previously their packages were not created due to a hack in debian/rules, that got removed in commit 9631d933fbc2ec99874. This change complements that commit. --- debian/autobake-deb.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 1a709eb8284..2158d0d0e29 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -87,6 +87,12 @@ then sed '/Package: mariadb-plugin-aws-key-management-10.3/,+15d' -i debian/control fi +# Mroonga, TokuDB never built on Travis CI anyway, see build flags above +if [[ $TRAVIS ]] +then + sed -i -e "/Package: mariadb-plugin-tokudb/,+17d" debian/control + sed -i -e "/Package: mariadb-plugin-mroonga/,+16d" debian/control +fi # Adjust changelog, add new version echo "Incrementing changelog and starting build scripts" From 094938255d1926f16b0f74bcbd7c3e967eaef2ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 12:05:30 +0000 Subject: [PATCH 64/82] Deb: sync architecture restrictions from packaging in Debian official --- debian/control | 4 ++-- debian/rules | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/debian/control b/debian/control index fe9a10a2cb8..bbea53788ca 100644 --- a/debian/control +++ b/debian/control @@ -500,7 +500,7 @@ Description: OQGraph storage engine for MariaDB This package contains the OQGraph plugin for MariaDB. Package: mariadb-plugin-tokudb -Architecture: any +Architecture: amd64 Depends: mariadb-server-10.3 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} @@ -519,7 +519,7 @@ Description: TokuDB storage engine for MariaDB This package contains the TokuDB plugin for MariaDB. Package: mariadb-plugin-mroonga -Architecture: any +Architecture: any-alpha any-amd64 any-arm any-arm64 any-i386 any-ia64 any-mips64el any-mips64r6el any-mipsel any-mipsr6el any-nios2 any-powerpcel any-ppc64el any-sh3 any-sh4 any-tilegx Depends: mariadb-server-10.3 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} diff --git a/debian/rules b/debian/rules index 76433d47576..456af8b0664 100755 --- a/debian/rules +++ b/debian/rules @@ -39,7 +39,7 @@ else endif # Ignore test suite exit code on unstable platforms -ifneq (,$(filter $(ARCH), mips mipsel)) +ifneq (,$(filter $(ARCH),mips mipsel mips64el alpha powerpc sh4 hurd-i386 sparc64 kfreebsd-i386 kfreebsd-amd64)) TESTSUITE_FAIL_CMD:=true else TESTSUITE_FAIL_CMD:=exit 1 @@ -114,11 +114,6 @@ override_dh_auto_install: dh_testdir dh_testroot -# Skip TokuDB if arch is not amd64 -ifneq ($(ARCH), amd64) - sed -i -e "/Package: mariadb-plugin-tokudb/,+18d" debian/control -endif - # If libthrift-dev was available (manually installed, as it is # not in Debian) and ha_cassandra.so was thus built, create package, # otherwise skip it. From 846517d9b704a91c557e0992f149f5890ae796ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 09:46:09 +0000 Subject: [PATCH 65/82] Deb: Recover include/mariadb wrongly removed in commit 9b10b4181f70634c The libmariadb-dev package must include this directory so that downstream developers can intentionally include MariaDB libraries as they want. This matches the contents of the official Debian libmariadb-dev package. --- debian/libmariadb-dev.install | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index e62aad1b43f..7aac84d0000 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,5 +1,6 @@ usr/bin/mysql_config usr/include/mysql +usr/include/mariadb usr/lib/*/libmariadb.so usr/lib/*/libmysqlclient.so usr/lib/*/libmariadbclient.so From 9719fb4beffd0cec057be63f43beb018099c77d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 10:46:04 +0000 Subject: [PATCH 66/82] Deb: Re-rename *.files into *.install The correct format for new Debian packages is to define install lists as *.install files. This has already been fixed for 10.3, but it when merging commit c5a525bda2b64dd008598afa51f3b7c6347e24fc from 10.2, this change was overlooked and a file of the old format slipped in. --- debian/{mariadb-backup-10.3.files => mariadb-backup-10.3.install} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename debian/{mariadb-backup-10.3.files => mariadb-backup-10.3.install} (100%) diff --git a/debian/mariadb-backup-10.3.files b/debian/mariadb-backup-10.3.install similarity index 100% rename from debian/mariadb-backup-10.3.files rename to debian/mariadb-backup-10.3.install From 503fcfb80f62da4a02912404ca724868edc607e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 10:51:03 +0000 Subject: [PATCH 67/82] Deb: Re-remove unnecessary version suffixes from package names There is no point in having version suffixes in plugin packages. They will via their control file anyway depend on the correct server or client version. Having version suffices has no benefit and it just complicates upgrades when a new major version name package needs to replace all possible previous version. This issue was overlooked when merging commit 844a5759462d11d5e4ae83d951b796647621e2d0 from 10.2 into 10.3, so it needs to be manually fixed now. --- debian/autobake-deb.sh | 5 ++++- debian/control | 2 +- ...0.2.install => mariadb-plugin-aws-key-management.install} | 0 3 files changed, 5 insertions(+), 2 deletions(-) rename debian/{mariadb-plugin-aws-key-management-10.2.install => mariadb-plugin-aws-key-management.install} (100%) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 2158d0d0e29..5948229ac28 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -82,9 +82,12 @@ if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]] || [[ $TRAVIS ]] then sed '/Package: mariadb-plugin-rocksdb/,+13d' -i debian/control fi + +# AWS SDK requires c++11 -capable compiler +# Minimal supported versions are g++ 4.8 and clang 3.3. if [[ $GCCVERSION -lt 40800 ]] || [[ $TRAVIS ]] then - sed '/Package: mariadb-plugin-aws-key-management-10.3/,+15d' -i debian/control + sed '/Package: mariadb-plugin-aws-key-management/,+14d' -i debian/control fi # Mroonga, TokuDB never built on Travis CI anyway, see build flags above diff --git a/debian/control b/debian/control index bbea53788ca..8c0874648ce 100644 --- a/debian/control +++ b/debian/control @@ -626,7 +626,7 @@ Description: CrackLib Password Validation Plugin for MariaDB This password validation plugin uses cracklib to allow only sufficiently secure (as defined by cracklib) user passwords in MariaDB. -Package: mariadb-plugin-aws-key-management-10.3 +Package: mariadb-plugin-aws-key-management Section: database Architecture: any Breaks: mariadb-aws-key-management-10.1, diff --git a/debian/mariadb-plugin-aws-key-management-10.2.install b/debian/mariadb-plugin-aws-key-management.install similarity index 100% rename from debian/mariadb-plugin-aws-key-management-10.2.install rename to debian/mariadb-plugin-aws-key-management.install From 90f1f72622cec1c8407cfffc67a7ec94f9cbe6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 12:49:21 +0000 Subject: [PATCH 68/82] Deb: Add missing files to packages - tokuft_logprint should be shipped in the TokuDB plugin - mysql_install_db is needed for installing/upgrading the datadir so it must be in the server core package so users can roll their own upgrades --- debian/mariadb-plugin-tokudb.install | 1 + debian/mariadb-server-10.3.install | 1 - debian/mariadb-server-10.3.manpages | 1 - debian/mariadb-server-core-10.3.install | 1 + debian/mariadb-server-core-10.3.manpages | 1 + 5 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debian/mariadb-plugin-tokudb.install b/debian/mariadb-plugin-tokudb.install index 3c366ee0c44..36beb14885f 100644 --- a/debian/mariadb-plugin-tokudb.install +++ b/debian/mariadb-plugin-tokudb.install @@ -1,4 +1,5 @@ etc/mysql/conf.d/tokudb.cnf etc/mysql/mariadb.conf.d +usr/bin/tokuft_logprint usr/bin/tokuftdump usr/lib/mysql/plugin/ha_tokudb.so usr/share/doc/mariadb-server-10.3/README.md usr/share/doc/mariadb-plugin-tokudb/README.md diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index 604179c7dca..487837e9582 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -19,7 +19,6 @@ usr/bin/myisamchk usr/bin/myisamlog usr/bin/myisampack usr/bin/mysql_convert_table_format -usr/bin/mysql_install_db usr/bin/mysql_plugin usr/bin/mysql_secure_installation usr/bin/mysql_setpermission diff --git a/debian/mariadb-server-10.3.manpages b/debian/mariadb-server-10.3.manpages index 1825b7655da..ebe72c24592 100644 --- a/debian/mariadb-server-10.3.manpages +++ b/debian/mariadb-server-10.3.manpages @@ -14,7 +14,6 @@ debian/tmp/usr/share/man/man1/mysql_convert_table_format.1 debian/tmp/usr/share/man/man1/mysqld_multi.1 debian/tmp/usr/share/man/man1/mysqld_safe.1 debian/tmp/usr/share/man/man1/mysqlhotcopy.1 -debian/tmp/usr/share/man/man1/mysql_install_db.1 debian/tmp/usr/share/man/man1/mysql_plugin.1 debian/tmp/usr/share/man/man1/mysql_secure_installation.1 debian/tmp/usr/share/man/man1/mysql_setpermission.1 diff --git a/debian/mariadb-server-core-10.3.install b/debian/mariadb-server-core-10.3.install index d882bd53f52..6aadea0adc8 100644 --- a/debian/mariadb-server-core-10.3.install +++ b/debian/mariadb-server-core-10.3.install @@ -1,4 +1,5 @@ usr/bin/innochecksum +usr/bin/mysql_install_db usr/bin/mysql_upgrade usr/sbin/mysqld usr/share/mysql/charsets diff --git a/debian/mariadb-server-core-10.3.manpages b/debian/mariadb-server-core-10.3.manpages index 234108cde50..04187d4cdf2 100644 --- a/debian/mariadb-server-core-10.3.manpages +++ b/debian/mariadb-server-core-10.3.manpages @@ -1,3 +1,4 @@ debian/tmp/usr/share/man/man8/mysqld.8 debian/tmp/usr/share/man/man1/innochecksum.1 +debian/tmp/usr/share/man/man1/mysql_install_db.1 debian/tmp/usr/share/man/man1/mysql_upgrade.1 From 65db816c325eab6b28e05617b91ec6df9aecd0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 14:59:46 +0000 Subject: [PATCH 69/82] Deb: Split libmysqlclient19/20 compat links into a separate package We already ship libmysqlclient18 that replaces any existing package with the same name and install symlinks that point to out libmariadb.so. Instead of creating more packages, we add just one more that does the same and provides both libmysqlclient19 and -20 replacements symlinks. This is a clean solution, because systems that only use (and are built against) libmariadb will not need them and if somebody has done a mistake in the linker parameters, they will spot it and can easily update the filename to libmariadb. --- debian/control | 27 +++++++++++++++++++++------ debian/libmariadb3-compat.install | 2 ++ debian/libmariadb3.install | 2 -- 3 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 debian/libmariadb3-compat.install diff --git a/debian/control b/debian/control index 8c0874648ce..bca2393a1aa 100644 --- a/debian/control +++ b/debian/control @@ -50,10 +50,6 @@ Conflicts: mariadb-galera-server-10.0 (<< 10.0.5), mariadb-server-5.3, mariadb-server-5.5 (<< 5.5.33), libmariadbclient18 (<< 10.2.0) -Replaces: libmysqlclient19, - libmysqlclient20 -Provides: libmysqlclient19, - libmysqlclient20 Description: MariaDB database client library MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -62,6 +58,25 @@ Description: MariaDB database client library . This package includes the client library. +Package: libmariadb3-compat +Architecture: any +Section: libs +Depends: mariadb-common, libmariadb3, ${misc:Depends}, ${shlibs:Depends} +Breaks: libmysqlclient19, + libmysqlclient20 +Replaces: libmysqlclient19, + libmysqlclient20 +Provides: libmysqlclient19, + libmysqlclient20 +Description: MariaDB database client library MySQL compat package + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes the client runtime libraries that simulate and replace + the equivalents found in MySQL 5.6 and 5.7 (mysqlclient19 and 20). + Package: libmariadbclient18 Section: libs Architecture: any @@ -74,7 +89,7 @@ Description: Virtual package to satisfy external depends language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3 + This package provides compatibility symlinks for libmariadb3. Package: libmysqlclient18 Section: libs @@ -88,7 +103,7 @@ Description: Virtual package to satisfy external depends language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3 + This package provides compatibility symlinks for libmariadb3. Package: libmariadb-dev Architecture: any diff --git a/debian/libmariadb3-compat.install b/debian/libmariadb3-compat.install new file mode 100644 index 00000000000..4e0d631d6c8 --- /dev/null +++ b/debian/libmariadb3-compat.install @@ -0,0 +1,2 @@ +usr/lib/*/libmysqlclient.so.19 +usr/lib/*/libmysqlclient.so.20 diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install index 58e2bd1dcb2..55e2f3f4193 100644 --- a/debian/libmariadb3.install +++ b/debian/libmariadb3.install @@ -1,5 +1,3 @@ -usr/lib/*/libmysqlclient.so.19 -usr/lib/*/libmysqlclient.so.20 usr/lib/*/libmariadb.so.* usr/lib/mysql/plugin/dialog.so usr/lib/mysql/plugin/mysql_clear_password.so From 9ef2268eda401f65fc1a93cad37b9382a91b6f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 15:48:17 +0000 Subject: [PATCH 70/82] Deb: Simplify libmariadb.so.3 links No path is required. These files always reside in the same directory anyway. --- debian/rules | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debian/rules b/debian/rules index 456af8b0664..3a9ff27a825 100755 --- a/debian/rules +++ b/debian/rules @@ -148,13 +148,13 @@ override_dh_auto_install: install -D -m 644 debian/mariadb-server-10.3.py $(TMP)/usr/share/apport/package-hooks/source_mariadb-10.3.py # Install libmariadbclient18 compatibility links - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so.18 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so.18 # Install libmysqlclientclientXX compatibility links - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.18 - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.19 - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.20 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.18 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.19 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.20 touch $@ From be59851ad98c21cd388a3a5ff6d088602291c6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 15:11:16 +0000 Subject: [PATCH 71/82] Deb: Match official Debian package libmariadb-dev-compat Using this package any source that expects MySQL Client libraries should build without changes. Sources that expect MariaDB Connector C should build using only libmariadb-dev. Current MariaDB Connector/C does not build quite identically in this source tree compared to how it build in Debian official. The remaining differences are tracked in MDEV-14921. --- debian/control | 42 +++++++++++++++++++++------- debian/libmariadb-dev-compat.install | 4 +++ debian/libmariadb-dev-compat.links | 2 ++ debian/libmariadb-dev.install | 4 +-- debian/libmariadb-dev.links | 2 ++ debian/libmariadbd-dev.install | 1 - 6 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 debian/libmariadb-dev-compat.install create mode 100644 debian/libmariadb-dev-compat.links create mode 100644 debian/libmariadb-dev.links diff --git a/debian/control b/debian/control index bca2393a1aa..f86849e5741 100644 --- a/debian/control +++ b/debian/control @@ -112,22 +112,44 @@ Depends: libmariadb3 (= ${binary:Version}), zlib1g-dev, ${misc:Depends}, ${shlibs:Depends} -Breaks: libmariadbclient-dev, libmysqlclient-dev -Replaces: libmariadbclient-dev, libmysqlclient-dev -Conflicts: libmariadbclient16-dev, - libmysqlclient10-dev, - libmysqlclient12-dev, - libmysqlclient14-dev, - libmysqlclient15-dev, - libmysqlclient16-dev -Provides: libmariadbclient-dev, libmariadbclient-dev-compat, libmysqlclient-dev +Breaks: libmariadbclient-dev +Replaces: libmariadbclient-dev +Conflicts: libmariadbclient16-dev +Provides: libmariadbclient-dev Description: MariaDB database development files MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package includes development libraries and header files. + This package includes development libraries and header files. To allow sources + expecting the MariaDB Connector/C to build. Sources that expect the MySQL + Client libraries should use files from the libmariadb-dev-compat package. + +Package: libmariadb-dev-compat +Architecture: any +Multi-Arch: same +Section: libdevel +Priority: extra +Depends: libmariadb-dev (= ${binary:Version}), ${misc:Depends} +Conflicts: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev, + libmysqlclient10-dev, + libmysqlclient12-dev, + libmysqlclient14-dev, + libmysqlclient15-dev, + libmysqlclient16-dev +Provides: libmariadbclient-dev-compat, libmariadb-client-lgpl-dev-compat, libmysqlclient-dev +Replaces: libmariadbclient-dev-compat, libmariadb-client-lgpl-dev-compat, libmysqlclient-dev +Description: MariaDB Connector/C, compatibility symlinks + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes compatibility symlinks to allow sources expecting the + MySQL client libraries to be built against MariaDB Connector/C. Package: libmariadbd19 Architecture: any diff --git a/debian/libmariadb-dev-compat.install b/debian/libmariadb-dev-compat.install new file mode 100644 index 00000000000..7fe10c50f69 --- /dev/null +++ b/debian/libmariadb-dev-compat.install @@ -0,0 +1,4 @@ +usr/lib/*/libmysqlclient.a +usr/lib/*/libmysqlclient.so +usr/lib/*/libmysqlclient_r.a +usr/lib/*/libmysqlclient_r.so diff --git a/debian/libmariadb-dev-compat.links b/debian/libmariadb-dev-compat.links new file mode 100644 index 00000000000..50798417681 --- /dev/null +++ b/debian/libmariadb-dev-compat.links @@ -0,0 +1,2 @@ +usr/bin/mariadb_config usr/bin/mysql_config +usr/share/pkgconfig/mariadb.pc usr/share/pkgconfig/mysqlclient.pc diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index 7aac84d0000..ae26a1e10e9 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,8 +1,6 @@ -usr/bin/mysql_config +usr/bin/mariadb_config usr/include/mysql -usr/include/mariadb usr/lib/*/libmariadb.so -usr/lib/*/libmysqlclient.so usr/lib/*/libmariadbclient.so usr/lib/*/libmariadbclient.a usr/lib/*/libmysqlservices.a diff --git a/debian/libmariadb-dev.links b/debian/libmariadb-dev.links new file mode 100644 index 00000000000..14d60027b8a --- /dev/null +++ b/debian/libmariadb-dev.links @@ -0,0 +1,2 @@ +usr/include/mysql usr/include/mariadb +usr/share/man/man1/mysql_config.1.gz usr/share/man/man1/mariadb_config.1.gz diff --git a/debian/libmariadbd-dev.install b/debian/libmariadbd-dev.install index 6fe225b15ba..4608bf6e838 100644 --- a/debian/libmariadbd-dev.install +++ b/debian/libmariadbd-dev.install @@ -1,3 +1,2 @@ -usr/bin/mariadb_config usr/lib/*/libmysqld.a usr/lib/*/libmysqld.so From 5b3bf57be251929e2a7e32ab4499559934c6ccbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 17:58:41 +0000 Subject: [PATCH 72/82] Deb: Install missing man pages --- debian/mariadb-plugin-tokudb.manpages | 2 ++ debian/mariadb-server-10.3.manpages | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 debian/mariadb-plugin-tokudb.manpages diff --git a/debian/mariadb-plugin-tokudb.manpages b/debian/mariadb-plugin-tokudb.manpages new file mode 100644 index 00000000000..102a593b809 --- /dev/null +++ b/debian/mariadb-plugin-tokudb.manpages @@ -0,0 +1,2 @@ +debian/tmp/usr/share/man/man1/tokuft_logdump.1 +debian/tmp/usr/share/man/man1/tokuftdump.1 diff --git a/debian/mariadb-server-10.3.manpages b/debian/mariadb-server-10.3.manpages index ebe72c24592..a05b695585e 100644 --- a/debian/mariadb-server-10.3.manpages +++ b/debian/mariadb-server-10.3.manpages @@ -3,6 +3,9 @@ debian/tmp/usr/share/man/man1/aria_dump_log.1 debian/tmp/usr/share/man/man1/aria_ftdump.1 debian/tmp/usr/share/man/man1/aria_pack.1 debian/tmp/usr/share/man/man1/aria_read_log.1 +debian/tmp/usr/share/man/man1/galera_new_cluster.1 +debian/tmp/usr/share/man/man1/galera_recovery.1 +debian/tmp/usr/share/man/man1/mariadb-service-convert.1 debian/tmp/usr/share/man/man1/msql2mysql.1 debian/tmp/usr/share/man/man1/myisamchk.1 debian/tmp/usr/share/man/man1/myisam_ftdump.1 @@ -18,7 +21,13 @@ debian/tmp/usr/share/man/man1/mysql_plugin.1 debian/tmp/usr/share/man/man1/mysql_secure_installation.1 debian/tmp/usr/share/man/man1/mysql_setpermission.1 debian/tmp/usr/share/man/man1/mysql_tzinfo_to_sql.1 +debian/tmp/usr/share/man/man1/mysqld_safe_helper.1 debian/tmp/usr/share/man/man1/perror.1 debian/tmp/usr/share/man/man1/replace.1 debian/tmp/usr/share/man/man1/resolveip.1 debian/tmp/usr/share/man/man1/resolve_stack_dump.1 +debian/tmp/usr/share/man/man1/wsrep_sst_common.1 +debian/tmp/usr/share/man/man1/wsrep_sst_mysqldump.1 +debian/tmp/usr/share/man/man1/wsrep_sst_rsync.1 +debian/tmp/usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 +debian/tmp/usr/share/man/man1/wsrep_sst_xtrabackup.1 From ed00fd2bced5a7d52e3a26f95aa8acdb45eb49da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 20:04:09 +0000 Subject: [PATCH 73/82] Deb: switch from usr/include/mysql to usr/include/mariadb Matching libmariadb-dev package contents in official Debian repositories, the MariaDB Connector/C files should go into a folder using the 'mariadb' name. For compatibility with sources that expect to find 'mysql' stuff, create a symlink from include/mysql to include/mariadb. --- cmake/install_layout.cmake | 2 +- debian/libmariadb-dev-compat.links | 1 + debian/libmariadb-dev.install | 2 +- debian/libmariadb-dev.links | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/install_layout.cmake b/cmake/install_layout.cmake index 5484691e94a..63e2419951b 100644 --- a/cmake/install_layout.cmake +++ b/cmake/install_layout.cmake @@ -176,7 +176,7 @@ SET(INSTALL_SYSCONF2DIR_DEB "/etc/mysql/conf.d") SET(INSTALL_LIBDIR_DEB "lib") SET(INSTALL_PLUGINDIR_DEB "lib/mysql/plugin") # -SET(INSTALL_INCLUDEDIR_DEB "include/mysql") +SET(INSTALL_INCLUDEDIR_DEB "include/mariadb") # SET(INSTALL_DOCDIR_DEB "share/doc") SET(INSTALL_DOCREADMEDIR_DEB "share/doc") diff --git a/debian/libmariadb-dev-compat.links b/debian/libmariadb-dev-compat.links index 50798417681..11e0dbce225 100644 --- a/debian/libmariadb-dev-compat.links +++ b/debian/libmariadb-dev-compat.links @@ -1,2 +1,3 @@ usr/bin/mariadb_config usr/bin/mysql_config +usr/include/mariadb usr/include/mysql usr/share/pkgconfig/mariadb.pc usr/share/pkgconfig/mysqlclient.pc diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index ae26a1e10e9..5025ae4a47c 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,5 +1,5 @@ usr/bin/mariadb_config -usr/include/mysql +usr/include/mariadb usr/lib/*/libmariadb.so usr/lib/*/libmariadbclient.so usr/lib/*/libmariadbclient.a diff --git a/debian/libmariadb-dev.links b/debian/libmariadb-dev.links index 14d60027b8a..6a7bf707ca7 100644 --- a/debian/libmariadb-dev.links +++ b/debian/libmariadb-dev.links @@ -1,2 +1 @@ -usr/include/mysql usr/include/mariadb usr/share/man/man1/mysql_config.1.gz usr/share/man/man1/mariadb_config.1.gz From a41192def8ab67f4420e437ebd26646023a9bcca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 20:27:40 +0000 Subject: [PATCH 74/82] Deb: Fix spelling etc Lintian complaints --- debian/control | 15 +++++++++------ debian/mariadb-client-10.3.README.Debian | 2 +- debian/mariadb-server-10.3.README.Debian | 8 ++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/debian/control b/debian/control index f86849e5741..680b6e7de30 100644 --- a/debian/control +++ b/debian/control @@ -83,13 +83,14 @@ Architecture: any Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} Replaces: libmariadbclient18 Provides: libmariadbclient18 -Description: Virtual package to satisfy external depends +Description: Virtual package to satisfy external libmariadbclient18 depends MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3. + This package provides compatibility symlinks for binaries that expect to find + libmariadbclient.so.18 will automatically use libmariadb.so.3 instead. Package: libmysqlclient18 Section: libs @@ -97,13 +98,14 @@ Architecture: any Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} Replaces: libmysqlclient18 Provides: libmysqlclient18 -Description: Virtual package to satisfy external depends +Description: Virtual package to satisfy external libmysqlclient18 depends MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3. + This package provides compatibility symlinks for binaries that expect to find + libmysqlclient.so.18 will automatically use libmariadb.so.3 instead. Package: libmariadb-dev Architecture: any @@ -626,6 +628,7 @@ Replaces: mariadb-gssapi-server-10.1, mariadb-gssapi-server-10.2, mariadb-gssapi-server-10.3 Description: GSSAPI authentication plugin for MariaDB server + This package contains the server components. Package: mariadb-plugin-gssapi-client Architecture: any @@ -640,9 +643,9 @@ Replaces: mariadb-gssapi-client-10.1, mariadb-gssapi-client-10.2, mariadb-gssapi-client-10.3 Description: GSSAPI authentication plugin for MariaDB client + This package contains the client components. Package: mariadb-backup-10.3 -Section: database Architecture: any Breaks: mariadb-backup-10.1, mariadb-backup-10.2 @@ -652,6 +655,7 @@ Depends: mariadb-server-10.3, ${misc:Depends}, ${shlibs:Depends} Description: Backup tool for MariaDB server + This backup tool is guaranteed to be compatible with MariaDB. Package: mariadb-plugin-cracklib-password-check Architecture: any @@ -664,7 +668,6 @@ Description: CrackLib Password Validation Plugin for MariaDB sufficiently secure (as defined by cracklib) user passwords in MariaDB. Package: mariadb-plugin-aws-key-management -Section: database Architecture: any Breaks: mariadb-aws-key-management-10.1, mariadb-aws-key-management-10.2 diff --git a/debian/mariadb-client-10.3.README.Debian b/debian/mariadb-client-10.3.README.Debian index b245638f9c9..64f0f509951 100644 --- a/debian/mariadb-client-10.3.README.Debian +++ b/debian/mariadb-client-10.3.README.Debian @@ -1,4 +1,4 @@ FAQ: -Q: My completition is gone, why? +Q: My completion is gone, why? A: You have "no-auto-rehash" in the "[mysql]" section of /etc/mysql/my.cnf! diff --git a/debian/mariadb-server-10.3.README.Debian b/debian/mariadb-server-10.3.README.Debian index be2e33d705d..ca81cc357c4 100644 --- a/debian/mariadb-server-10.3.README.Debian +++ b/debian/mariadb-server-10.3.README.Debian @@ -42,7 +42,7 @@ https://mariadb.com/kb It is strongly recommended you create an admin users for your database adminstration needs. -If your your local unix account is the one you want to have local super user +If your local unix account is the one you want to have local super user access on your database with you can create the following account that will only work for the local unix user connecting to the database locally. @@ -57,14 +57,14 @@ the DB server over the network: sudo /usr/bin/mysql -e "GRANT ALL ON *.* TO 'USERNAME'@'%' IDENTIFIED BY 'password' WITH GRANT OPTION" -Scripts should run as a user have have the required grants and be identified via unix_socket. +Scripts should run as a user have the required grants and be identified via unix_socket. If you are too tired to type the password in every time and unix_socket auth doesn't suit your needs, you can store it in the file $HOME/.my.cnf. It should -be chmod 0600 (-rw------- username username .my.cnf) to ensure that nobody else +be chmod 0600 (-rw------- username usergroup .my.cnf) to ensure that nobody else can read it. Every other configuration parameter can be stored there, too. -For more information in the MariaDB manual in/usr/share/doc/mariadb-doc or +For more information in the MariaDB manual in/usr/share/doc/mariadb-doc or https://mariadb.com/kb/en/configuring-mariadb-with-mycnf/. ATTENTION: It is necessary, that a ~/.my.cnf from root always contains a "user" From 711e8c56346d0f9a4717ea2b2e0c7a98b6c247a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 13:06:14 +0000 Subject: [PATCH 75/82] MDEV-12642: Build deb source packages on buildbot, just not on Travis-CI --- debian/autobake-deb.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 5948229ac28..fca3482e64b 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -111,12 +111,20 @@ dch -b -D ${CODENAME} -v "${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automatic build echo "Creating package version ${UPSTREAM}${PATCHLEVEL}~${CODENAME} ... " +# On Travis CI, use -b to build binary only packages as there is no need to +# waste time on generating the source package. +if [[ $TRAVIS ]] +then + BUILDPACKAGE_FLAGS="-b" +fi + # Build the package # Pass -I so that .git and other unnecessary temporary and source control files # will be ignored by dpkg-source when creating the tar.gz source package. -# Use -b to build binary only packages as there is no need to waste time on -# generating the source package. -fakeroot dpkg-buildpackage -us -uc -I -b +fakeroot dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS + +# If the step above fails due to missing dependencies, you can manually run +# sudo mk-build-deps debian/control -r -i # Don't log package contents on Travis-CI to save time and log size if [[ ! $TRAVIS ]] From dbc9cfa52b822ebd0542faf09e0a561b638a637b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 19:35:01 +0000 Subject: [PATCH 76/82] Deb: .manpages files should only contain man pages unique to deb packages All man pages that stem from the actual sources should be installed along everything else in .install files. --- debian/libmariadb-dev.install | 1 + debian/libmariadb-dev.manpages | 1 - debian/mariadb-client-10.3.install | 10 +++++++ debian/mariadb-client-10.3.manpages | 10 ------- debian/mariadb-client-core-10.3.install | 2 ++ debian/mariadb-client-core-10.3.manpages | 2 -- debian/mariadb-plugin-tokudb.install | 2 ++ debian/mariadb-plugin-tokudb.manpages | 2 -- debian/mariadb-server-10.3.install | 33 ++++++++++++++++++++++++ debian/mariadb-server-10.3.manpages | 33 ------------------------ debian/mariadb-server-core-10.3.install | 4 +++ debian/mariadb-server-core-10.3.manpages | 4 --- debian/mariadb-test.install | 4 +++ debian/mariadb-test.manpages | 4 --- 14 files changed, 56 insertions(+), 56 deletions(-) delete mode 100644 debian/libmariadb-dev.manpages delete mode 100644 debian/mariadb-client-core-10.3.manpages delete mode 100644 debian/mariadb-plugin-tokudb.manpages delete mode 100644 debian/mariadb-server-10.3.manpages delete mode 100644 debian/mariadb-server-core-10.3.manpages delete mode 100644 debian/mariadb-test.manpages diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index 5025ae4a47c..f33febf0e50 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -6,3 +6,4 @@ usr/lib/*/libmariadbclient.a usr/lib/*/libmysqlservices.a usr/share/aclocal/mysql.m4 usr/share/pkgconfig/mariadb.pc +usr/share/man/man1/mysql_config.1 diff --git a/debian/libmariadb-dev.manpages b/debian/libmariadb-dev.manpages deleted file mode 100644 index 3aac7f4b4a9..00000000000 --- a/debian/libmariadb-dev.manpages +++ /dev/null @@ -1 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql_config.1 diff --git a/debian/mariadb-client-10.3.install b/debian/mariadb-client-10.3.install index 271f2a2f417..75ee09db783 100644 --- a/debian/mariadb-client-10.3.install +++ b/debian/mariadb-client-10.3.install @@ -10,3 +10,13 @@ usr/bin/mysqldumpslow usr/bin/mysqlimport usr/bin/mysqlshow usr/bin/mysqlslap +usr/share/man/man1/mysqlaccess.1 +usr/share/man/man1/mysqladmin.1 +usr/share/man/man1/mysqldump.1 +usr/share/man/man1/mysqldumpslow.1 +usr/share/man/man1/mysql_find_rows.1 +usr/share/man/man1/mysql_fix_extensions.1 +usr/share/man/man1/mysqlimport.1 +usr/share/man/man1/mysqlshow.1 +usr/share/man/man1/mysqlslap.1 +usr/share/man/man1/mysql_waitpid.1 diff --git a/debian/mariadb-client-10.3.manpages b/debian/mariadb-client-10.3.manpages index 504619785bc..6f3e2bc188c 100644 --- a/debian/mariadb-client-10.3.manpages +++ b/debian/mariadb-client-10.3.manpages @@ -1,12 +1,2 @@ debian/additions/innotop/innotop.1 -debian/tmp/usr/share/man/man1/mysqlaccess.1 -debian/tmp/usr/share/man/man1/mysqladmin.1 -debian/tmp/usr/share/man/man1/mysqldump.1 -debian/tmp/usr/share/man/man1/mysqldumpslow.1 -debian/tmp/usr/share/man/man1/mysql_find_rows.1 -debian/tmp/usr/share/man/man1/mysql_fix_extensions.1 -debian/tmp/usr/share/man/man1/mysqlimport.1 debian/additions/mysqlreport.1 -debian/tmp/usr/share/man/man1/mysqlshow.1 -debian/tmp/usr/share/man/man1/mysqlslap.1 -debian/tmp/usr/share/man/man1/mysql_waitpid.1 diff --git a/debian/mariadb-client-core-10.3.install b/debian/mariadb-client-core-10.3.install index 6308d769930..a2781309439 100644 --- a/debian/mariadb-client-core-10.3.install +++ b/debian/mariadb-client-core-10.3.install @@ -1,2 +1,4 @@ usr/bin/mysql usr/bin/mysqlcheck +usr/share/man/man1/mysql.1 +usr/share/man/man1/mysqlcheck.1 diff --git a/debian/mariadb-client-core-10.3.manpages b/debian/mariadb-client-core-10.3.manpages deleted file mode 100644 index 2be91d81f9a..00000000000 --- a/debian/mariadb-client-core-10.3.manpages +++ /dev/null @@ -1,2 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql.1 -debian/tmp/usr/share/man/man1/mysqlcheck.1 diff --git a/debian/mariadb-plugin-tokudb.install b/debian/mariadb-plugin-tokudb.install index 36beb14885f..d15db03b3e5 100644 --- a/debian/mariadb-plugin-tokudb.install +++ b/debian/mariadb-plugin-tokudb.install @@ -3,3 +3,5 @@ usr/bin/tokuft_logprint usr/bin/tokuftdump usr/lib/mysql/plugin/ha_tokudb.so usr/share/doc/mariadb-server-10.3/README.md usr/share/doc/mariadb-plugin-tokudb/README.md +usr/share/man/man1/tokuft_logdump.1 +usr/share/man/man1/tokuftdump.1 diff --git a/debian/mariadb-plugin-tokudb.manpages b/debian/mariadb-plugin-tokudb.manpages deleted file mode 100644 index 102a593b809..00000000000 --- a/debian/mariadb-plugin-tokudb.manpages +++ /dev/null @@ -1,2 +0,0 @@ -debian/tmp/usr/share/man/man1/tokuft_logdump.1 -debian/tmp/usr/share/man/man1/tokuftdump.1 diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index 487837e9582..ac1ab6dd990 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -65,3 +65,36 @@ usr/share/mysql/mysql_system_tables.sql usr/share/mysql/mysql_system_tables_data.sql usr/share/mysql/mysql_test_data_timezone.sql usr/share/mysql/wsrep_notify +usr/share/man/man1/aria_chk.1 +usr/share/man/man1/aria_dump_log.1 +usr/share/man/man1/aria_ftdump.1 +usr/share/man/man1/aria_pack.1 +usr/share/man/man1/aria_read_log.1 +usr/share/man/man1/galera_new_cluster.1 +usr/share/man/man1/galera_recovery.1 +usr/share/man/man1/mariadb-service-convert.1 +usr/share/man/man1/msql2mysql.1 +usr/share/man/man1/myisamchk.1 +usr/share/man/man1/myisam_ftdump.1 +usr/share/man/man1/myisamlog.1 +usr/share/man/man1/myisampack.1 +usr/share/man/man1/my_print_defaults.1 +usr/share/man/man1/mysqlbinlog.1 +usr/share/man/man1/mysql_convert_table_format.1 +usr/share/man/man1/mysqld_multi.1 +usr/share/man/man1/mysqld_safe.1 +usr/share/man/man1/mysqlhotcopy.1 +usr/share/man/man1/mysql_plugin.1 +usr/share/man/man1/mysql_secure_installation.1 +usr/share/man/man1/mysql_setpermission.1 +usr/share/man/man1/mysql_tzinfo_to_sql.1 +usr/share/man/man1/mysqld_safe_helper.1 +usr/share/man/man1/perror.1 +usr/share/man/man1/replace.1 +usr/share/man/man1/resolveip.1 +usr/share/man/man1/resolve_stack_dump.1 +usr/share/man/man1/wsrep_sst_common.1 +usr/share/man/man1/wsrep_sst_mysqldump.1 +usr/share/man/man1/wsrep_sst_rsync.1 +usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 +usr/share/man/man1/wsrep_sst_xtrabackup.1 diff --git a/debian/mariadb-server-10.3.manpages b/debian/mariadb-server-10.3.manpages deleted file mode 100644 index a05b695585e..00000000000 --- a/debian/mariadb-server-10.3.manpages +++ /dev/null @@ -1,33 +0,0 @@ -debian/tmp/usr/share/man/man1/aria_chk.1 -debian/tmp/usr/share/man/man1/aria_dump_log.1 -debian/tmp/usr/share/man/man1/aria_ftdump.1 -debian/tmp/usr/share/man/man1/aria_pack.1 -debian/tmp/usr/share/man/man1/aria_read_log.1 -debian/tmp/usr/share/man/man1/galera_new_cluster.1 -debian/tmp/usr/share/man/man1/galera_recovery.1 -debian/tmp/usr/share/man/man1/mariadb-service-convert.1 -debian/tmp/usr/share/man/man1/msql2mysql.1 -debian/tmp/usr/share/man/man1/myisamchk.1 -debian/tmp/usr/share/man/man1/myisam_ftdump.1 -debian/tmp/usr/share/man/man1/myisamlog.1 -debian/tmp/usr/share/man/man1/myisampack.1 -debian/tmp/usr/share/man/man1/my_print_defaults.1 -debian/tmp/usr/share/man/man1/mysqlbinlog.1 -debian/tmp/usr/share/man/man1/mysql_convert_table_format.1 -debian/tmp/usr/share/man/man1/mysqld_multi.1 -debian/tmp/usr/share/man/man1/mysqld_safe.1 -debian/tmp/usr/share/man/man1/mysqlhotcopy.1 -debian/tmp/usr/share/man/man1/mysql_plugin.1 -debian/tmp/usr/share/man/man1/mysql_secure_installation.1 -debian/tmp/usr/share/man/man1/mysql_setpermission.1 -debian/tmp/usr/share/man/man1/mysql_tzinfo_to_sql.1 -debian/tmp/usr/share/man/man1/mysqld_safe_helper.1 -debian/tmp/usr/share/man/man1/perror.1 -debian/tmp/usr/share/man/man1/replace.1 -debian/tmp/usr/share/man/man1/resolveip.1 -debian/tmp/usr/share/man/man1/resolve_stack_dump.1 -debian/tmp/usr/share/man/man1/wsrep_sst_common.1 -debian/tmp/usr/share/man/man1/wsrep_sst_mysqldump.1 -debian/tmp/usr/share/man/man1/wsrep_sst_rsync.1 -debian/tmp/usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 -debian/tmp/usr/share/man/man1/wsrep_sst_xtrabackup.1 diff --git a/debian/mariadb-server-core-10.3.install b/debian/mariadb-server-core-10.3.install index 6aadea0adc8..6244abd1193 100644 --- a/debian/mariadb-server-core-10.3.install +++ b/debian/mariadb-server-core-10.3.install @@ -27,3 +27,7 @@ usr/share/mysql/slovak usr/share/mysql/spanish usr/share/mysql/swedish usr/share/mysql/ukrainian +usr/share/man/man8/mysqld.8 +usr/share/man/man1/innochecksum.1 +usr/share/man/man1/mysql_install_db.1 +usr/share/man/man1/mysql_upgrade.1 diff --git a/debian/mariadb-server-core-10.3.manpages b/debian/mariadb-server-core-10.3.manpages deleted file mode 100644 index 04187d4cdf2..00000000000 --- a/debian/mariadb-server-core-10.3.manpages +++ /dev/null @@ -1,4 +0,0 @@ -debian/tmp/usr/share/man/man8/mysqld.8 -debian/tmp/usr/share/man/man1/innochecksum.1 -debian/tmp/usr/share/man/man1/mysql_install_db.1 -debian/tmp/usr/share/man/man1/mysql_upgrade.1 diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 605620dc28c..6c59a78c2d7 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -26,3 +26,7 @@ usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/purify.supp usr/share/mysql/mysql-test/suite.pm usr/share/mysql/mysql-test/valgrind.supp +usr/share/man/man1/mysql_client_test.1 +usr/share/man/man1/mysql_client_test_embedded.1 +usr/share/man/man1/mysqltest.1 +usr/share/man/man1/mysqltest_embedded.1 diff --git a/debian/mariadb-test.manpages b/debian/mariadb-test.manpages deleted file mode 100644 index 33ce8710ea7..00000000000 --- a/debian/mariadb-test.manpages +++ /dev/null @@ -1,4 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql_client_test.1 -debian/tmp/usr/share/man/man1/mysql_client_test_embedded.1 -debian/tmp/usr/share/man/man1/mysqltest.1 -debian/tmp/usr/share/man/man1/mysqltest_embedded.1 From b5edb4ca3a39c46e0109cf888aeb6ca54a6fc9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 16:56:55 +0000 Subject: [PATCH 77/82] Deb: Add missing files into correct packages and rest in not-installed There was a lot of files generated by the sources that were not installed in any package. This fixes most of those issues, but not all. Files still outside any package are in the not-installed, which is used by dh_install. Also make sure all configuration files are installed to the correct location that matches layout used by Debian official packages. --- debian/libmariadb3.install | 2 + debian/mariadb-common.install | 2 +- .../mariadb-plugin-aws-key-management.install | 3 +- debian/mariadb-plugin-connect.install | 2 +- debian/mariadb-plugin-mroonga.install | 2 + debian/mariadb-server-10.3.install | 1 + debian/mariadb-test.install | 3 + debian/not-installed | 127 ++++++++++++++++++ 8 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 debian/not-installed diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install index 55e2f3f4193..68d763ae404 100644 --- a/debian/libmariadb3.install +++ b/debian/libmariadb3.install @@ -1,3 +1,5 @@ usr/lib/*/libmariadb.so.* usr/lib/mysql/plugin/dialog.so usr/lib/mysql/plugin/mysql_clear_password.so +usr/lib/mysql/plugin/client_ed25519.so +usr/lib/mysql/plugin/sha256_password.so diff --git a/debian/mariadb-common.install b/debian/mariadb-common.install index 78dbc22b9f6..611c7d4d36e 100644 --- a/debian/mariadb-common.install +++ b/debian/mariadb-common.install @@ -1 +1 @@ -debian/additions/mariadb.cnf etc/mysql/conf.d +debian/additions/mariadb.cnf etc/mysql/ diff --git a/debian/mariadb-plugin-aws-key-management.install b/debian/mariadb-plugin-aws-key-management.install index ed966b4115d..655735cb8ac 100644 --- a/debian/mariadb-plugin-aws-key-management.install +++ b/debian/mariadb-plugin-aws-key-management.install @@ -1,2 +1,3 @@ usr/lib/mysql/plugin/aws_key_management.so -debian/additions/enable_encryption.preset etc/mysql/conf.d/ +debian/additions/enable_encryption.preset etc/mysql/mariadb.conf.d +etc/mysql/conf.d/aws_key_management.cnf etc/mysql/mariadb.conf.d diff --git a/debian/mariadb-plugin-connect.install b/debian/mariadb-plugin-connect.install index 8a7aee412df..22d73c7df05 100644 --- a/debian/mariadb-plugin-connect.install +++ b/debian/mariadb-plugin-connect.install @@ -1,2 +1,2 @@ -etc/mysql/conf.d/connect.cnf +etc/mysql/conf.d/connect.cnf etc/mysql/mariadb.conf.d usr/lib/mysql/plugin/ha_connect.so diff --git a/debian/mariadb-plugin-mroonga.install b/debian/mariadb-plugin-mroonga.install index c28fde2fd18..5cb81419e5e 100644 --- a/debian/mariadb-plugin-mroonga.install +++ b/debian/mariadb-plugin-mroonga.install @@ -1,3 +1,5 @@ usr/lib/mysql/plugin/ha_mroonga.so usr/share/mysql/mroonga/install.sql usr/share/mysql/mroonga/uninstall.sql +usr/share/mysql/mroonga/COPYING +usr/share/mysql/mroonga/AUTHORS diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index ac1ab6dd990..c0af11c5cc0 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -41,6 +41,7 @@ usr/bin/wsrep_sst_mariabackup usr/lib/mysql/plugin/auth_pam.so usr/lib/mysql/plugin/auth_socket.so usr/lib/mysql/plugin/file_key_management.so +usr/lib/mysql/plugin/auth_ed25519.so usr/lib/mysql/plugin/ha_archive.so usr/lib/mysql/plugin/ha_blackhole.so usr/lib/mysql/plugin/ha_federated.so diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 6c59a78c2d7..4a3ef436634 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -26,6 +26,9 @@ usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/purify.supp usr/share/mysql/mysql-test/suite.pm usr/share/mysql/mysql-test/valgrind.supp +usr/share/mysql/mysql-test/dgcov.pl +usr/share/man/man1/mysql-stress-test.pl.1 +usr/share/man/man1/mysql-test-run.pl.1 usr/share/man/man1/mysql_client_test.1 usr/share/man/man1/mysql_client_test_embedded.1 usr/share/man/man1/mysqltest.1 diff --git a/debian/not-installed b/debian/not-installed new file mode 100644 index 00000000000..c151cf753e6 --- /dev/null +++ b/debian/not-installed @@ -0,0 +1,127 @@ +lib/systemd/system/mariadb.service # Installed by rules file +lib/systemd/system/mariadb@.service # Installed by rules file +usr/bin/mysql_config # We already have the MariaDB variant +usr/bin/mysql_embedded # Huge 500 MB file. Not intended for distribution via any disto package. +usr/bin/mytop # Mytop is distributed from a separate source package +usr/lib/sysusers.d/sysusers.conf +usr/lib/tmpfiles.d/tmpfiles.conf +usr/lib/mysql/plugin/JavaWrappers.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless +usr/lib/mysql/plugin/JdbcInterface.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless +usr/share/doc/mariadb-server-10.3/COPYING +usr/share/doc/mariadb-server-10.3/COPYING.AGPLv3 +usr/share/doc/mariadb-server-10.3/COPYING.GPLv2 +usr/share/doc/mariadb-server-10.3/COPYING.thirdparty +usr/share/doc/mariadb-server-10.3/CREDITS +usr/share/doc/mariadb-server-10.3/EXCEPTIONS-CLIENT +usr/share/doc/mariadb-server-10.3/INSTALL-BINARY +usr/share/doc/mariadb-server-10.3/PATENTS +usr/share/doc/mariadb-server-10.3/README-wsrep +usr/share/groonga/COPYING +usr/share/groonga-normalizer-mysql/lgpl-2.0.txt +usr/share/groonga-normalizer-mysql/README.md +usr/share/groonga/README.md +usr/share/man/man1/my_safe_process.1 +usr/share/man/man1/mysql.server.1 +usr/share/mysql/binary-configure +usr/share/mysql/magic +usr/share/mysql/maria_add_gis_sp.sql +usr/share/mysql/mysqld_multi.server +usr/share/mysql/mysql-log-rotate +usr/share/mysql/mysql.server +usr/share/mysql/mysql-test/mtr # Already created by mariadb-test.links +usr/share/mysql/mysql-test/mysql-test-run # Already created by mariadb-test.links +usr/share/mysql/mysql_to_mariadb.sql +usr/share/mysql/policy/apparmor/README # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/apparmor/usr.sbin.mysqld # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/apparmor/usr.sbin.mysqld.local # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/selinux/mariadb-server.fc # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/mariadb-server.te # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/mariadb.te # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/README # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/systemd/mariadb.service # Installed by rules file +usr/share/mysql/systemd/mariadb@.service # Installed by rules file +usr/share/mysql/systemd/use_galera_new_cluster.conf +usr/share/mysql/wsrep.cnf +usr/sql-bench/bench-count-distinct +usr/sql-bench/bench-init.pl # SQL-bench is distributed from a separate source package +usr/sql-bench/compare-results +usr/sql-bench/copy-db +usr/sql-bench/crash-me +usr/sql-bench/Data/ATIS/aircraft.txt +usr/sql-bench/Data/ATIS/airline.txt +usr/sql-bench/Data/ATIS/airport_service.txt +usr/sql-bench/Data/ATIS/airport.txt +usr/sql-bench/Data/ATIS/city.txt +usr/sql-bench/Data/ATIS/class_of_service.txt +usr/sql-bench/Data/ATIS/code_description.txt +usr/sql-bench/Data/ATIS/compound_class.txt +usr/sql-bench/Data/ATIS/connect_leg.txt +usr/sql-bench/Data/ATIS/date_day.txt +usr/sql-bench/Data/ATIS/day_name.txt +usr/sql-bench/Data/ATIS/dual_carrier.txt +usr/sql-bench/Data/ATIS/fare.txt +usr/sql-bench/Data/ATIS/fconnection.txt +usr/sql-bench/Data/ATIS/flight_class.txt +usr/sql-bench/Data/ATIS/flight_day.txt +usr/sql-bench/Data/ATIS/flight_fare.txt +usr/sql-bench/Data/ATIS/flight.txt +usr/sql-bench/Data/ATIS/food_service.txt +usr/sql-bench/Data/ATIS/ground_service.txt +usr/sql-bench/Data/ATIS/month_name.txt +usr/sql-bench/Data/ATIS/restrict_carrier.txt +usr/sql-bench/Data/ATIS/restrict_class.txt +usr/sql-bench/Data/ATIS/restriction.txt +usr/sql-bench/Data/ATIS/state.txt +usr/sql-bench/Data/ATIS/stop1.txt +usr/sql-bench/Data/ATIS/stop.txt +usr/sql-bench/Data/ATIS/time_interval.txt +usr/sql-bench/Data/ATIS/time_zone.txt +usr/sql-bench/Data/ATIS/transport.txt +usr/sql-bench/Data/Wisconsin/onek.data +usr/sql-bench/Data/Wisconsin/tenk.data +usr/sql-bench/graph-compare-results +usr/sql-bench/innotest1 +usr/sql-bench/innotest1a +usr/sql-bench/innotest1b +usr/sql-bench/innotest2 +usr/sql-bench/innotest2a +usr/sql-bench/innotest2b +usr/sql-bench/limits/access.cfg +usr/sql-bench/limits/access_odbc.cfg +usr/sql-bench/limits/Adabas.cfg +usr/sql-bench/limits/db2.cfg +usr/sql-bench/limits/empress.cfg +usr/sql-bench/limits/frontbase.cfg +usr/sql-bench/limits/Informix.cfg +usr/sql-bench/limits/interbase.cfg +usr/sql-bench/limits/interbase-dialect1.cfg +usr/sql-bench/limits/interbase-dialect3.cfg +usr/sql-bench/limits/interbase-superserver.cfg +usr/sql-bench/limits/mimer.cfg +usr/sql-bench/limits/msql.cfg +usr/sql-bench/limits/ms-sql65.cfg +usr/sql-bench/limits/ms-sql.cfg +usr/sql-bench/limits/mysql-3.22.cfg +usr/sql-bench/limits/mysql-3.23.cfg +usr/sql-bench/limits/mysql-4.0.cfg +usr/sql-bench/limits/mysql-4.1.cfg +usr/sql-bench/limits/mysql.cfg +usr/sql-bench/limits/oracle.cfg +usr/sql-bench/limits/pg.cfg +usr/sql-bench/limits/solid.cfg +usr/sql-bench/limits/solid-nt4.cfg +usr/sql-bench/limits/sybase.cfg +usr/sql-bench/myisam.cnf +usr/sql-bench/README +usr/sql-bench/run-all-tests +usr/sql-bench/server-cfg +usr/sql-bench/test-alter-table +usr/sql-bench/test-ATIS +usr/sql-bench/test-big-tables +usr/sql-bench/test-connect +usr/sql-bench/test-create +usr/sql-bench/test-insert +usr/sql-bench/test-select +usr/sql-bench/test-table-elimination +usr/sql-bench/test-transactions +usr/sql-bench/test-wisconsin From 72768cf188536d2d29ff7e25f273be5981c01997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 20:27:22 +0000 Subject: [PATCH 78/82] Deb: Warn if sources contain a file that is not installed anywhere --- debian/rules | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index 3a9ff27a825..b4ef2e5da6d 100755 --- a/debian/rules +++ b/debian/rules @@ -177,7 +177,11 @@ override_dh_installcron-arch: get-orig-source: uscan --force-download --verbose +# If a file is not supposed to be included anywhere, add it to the not-installed +# file and document the reason. Note that dh_install supports the above mentioned +# white list file only starting from Debian Stretch and Ubuntu Xenial. +# To find more, grep build logs for 'but is not installed to anywhere'. %: - dh $@ --parallel --with dpatch --with systemd + dh $@ --parallel --with dpatch --with systemd --list-missing # vim: ts=8 From 3862d8bc89b9011c03738e3b48e444b692ae901a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 22:56:53 +0000 Subject: [PATCH 79/82] Deb: wrap-and-sort -a --- debian/control | 86 ++++++++++++------- debian/libmariadb-dev.install | 4 +- debian/libmariadb3.install | 2 +- debian/mariadb-client-10.3.docs | 2 +- debian/mariadb-client-10.3.install | 6 +- debian/mariadb-client-10.3.links | 4 +- .../mariadb-plugin-aws-key-management.install | 2 +- debian/mariadb-plugin-mroonga.install | 4 +- debian/mariadb-plugin-rocksdb.install | 2 +- debian/mariadb-server-10.3.install | 34 ++++---- debian/mariadb-server-core-10.3.install | 8 +- debian/mariadb-test.install | 14 +-- debian/mariadb-test.links | 2 +- debian/mysql-common.install | 2 +- 14 files changed, 99 insertions(+), 73 deletions(-) diff --git a/debian/control b/debian/control index 680b6e7de30..adc92a58087 100644 --- a/debian/control +++ b/debian/control @@ -13,17 +13,17 @@ Build-Depends: bison, libaio-dev [linux-any], libboost-dev, libcrack2-dev (>= 2.9.0), + libcurl3-dev, libjemalloc-dev (>= 3.0.0~) [linux-any], libjudy-dev, libkrb5-dev, - libcurl3-dev, libncurses5-dev (>= 5.0-6~), libnuma-dev, libpam0g-dev, libpcre3-dev (>= 2:8.35-3.2~), libreadline-gplv2-dev, - libssl-dev | libssl1.0-dev, libsnappy-dev, + libssl-dev | libssl1.0-dev, libsystemd-dev, libxml2-dev, lsb-release, @@ -41,15 +41,17 @@ Vcs-Browser: https://github.com/MariaDB/server/ Package: libmariadb3 Architecture: any Section: libs -Depends: mariadb-common, ${misc:Depends}, ${shlibs:Depends} -Conflicts: mariadb-galera-server-10.0 (<< 10.0.5), +Depends: mariadb-common, + ${misc:Depends}, + ${shlibs:Depends} +Conflicts: libmariadbclient18 (<< 10.2.0), + mariadb-galera-server-10.0 (<< 10.0.5), mariadb-galera-server-5.5 (<< 5.5.33), mariadb-server-10.0 (<< 10.0.5), mariadb-server-5.1, mariadb-server-5.2, mariadb-server-5.3, - mariadb-server-5.5 (<< 5.5.33), - libmariadbclient18 (<< 10.2.0) + mariadb-server-5.5 (<< 5.5.33) Description: MariaDB database client library MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -61,13 +63,16 @@ Description: MariaDB database client library Package: libmariadb3-compat Architecture: any Section: libs -Depends: mariadb-common, libmariadb3, ${misc:Depends}, ${shlibs:Depends} +Depends: libmariadb3, + mariadb-common, + ${misc:Depends}, + ${shlibs:Depends} Breaks: libmysqlclient19, - libmysqlclient20 + libmysqlclient20 Replaces: libmysqlclient19, - libmysqlclient20 + libmysqlclient20 Provides: libmysqlclient19, - libmysqlclient20 + libmysqlclient20 Description: MariaDB database client library MySQL compat package MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -80,7 +85,8 @@ Description: MariaDB database client library MySQL compat package Package: libmariadbclient18 Section: libs Architecture: any -Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} +Depends: libmariadb3 (= ${binary:Version}), + ${misc:Depends} Replaces: libmariadbclient18 Provides: libmariadbclient18 Description: Virtual package to satisfy external libmariadbclient18 depends @@ -95,7 +101,8 @@ Description: Virtual package to satisfy external libmariadbclient18 depends Package: libmysqlclient18 Section: libs Architecture: any -Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} +Depends: libmariadb3 (= ${binary:Version}), + ${misc:Depends} Replaces: libmysqlclient18 Provides: libmysqlclient18 Description: Virtual package to satisfy external libmysqlclient18 depends @@ -133,7 +140,8 @@ Architecture: any Multi-Arch: same Section: libdevel Priority: extra -Depends: libmariadb-dev (= ${binary:Version}), ${misc:Depends} +Depends: libmariadb-dev (= ${binary:Version}), + ${misc:Depends} Conflicts: libmariadb-client-lgpl-dev-compat, libmariadbclient-dev-compat, libmysqlclient-dev, @@ -142,8 +150,12 @@ Conflicts: libmariadb-client-lgpl-dev-compat, libmysqlclient14-dev, libmysqlclient15-dev, libmysqlclient16-dev -Provides: libmariadbclient-dev-compat, libmariadb-client-lgpl-dev-compat, libmysqlclient-dev -Replaces: libmariadbclient-dev-compat, libmariadb-client-lgpl-dev-compat, libmysqlclient-dev +Provides: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev +Replaces: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev Description: MariaDB Connector/C, compatibility symlinks MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -156,7 +168,8 @@ Description: MariaDB Connector/C, compatibility symlinks Package: libmariadbd19 Architecture: any Section: libs -Depends: ${misc:Depends}, ${shlibs:Depends} +Depends: ${misc:Depends}, + ${shlibs:Depends} Multi-Arch: same Breaks: libmariadbd-dev (<< ${source:Version}) Replaces: libmariadbd-dev (<< ${source:Version}) @@ -174,9 +187,9 @@ Section: libdevel Provides: libmysqld-dev Pre-Depends: ${misc:Pre-Depends} Depends: libmariadb-dev (= ${binary:Version}), - libmariadbd19 (= ${binary:Version}), - ${misc:Depends}, - ${shlibs:Depends} + libmariadbd19 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends} Breaks: libmysqld-dev Replaces: libmysqld-dev Description: MariaDB embedded database, development files @@ -189,7 +202,8 @@ Description: MariaDB embedded database, development files Package: mysql-common Architecture: all -Depends: ${misc:Depends}, ${shlibs:Depends} +Depends: ${misc:Depends}, + ${shlibs:Depends} Description: MariaDB database common files (e.g. /etc/mysql/my.cnf) MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -201,7 +215,9 @@ Description: MariaDB database common files (e.g. /etc/mysql/my.cnf) Package: mariadb-common Architecture: all -Depends: mysql-common, ${misc:Depends}, ${shlibs:Depends} +Depends: mysql-common, + ${misc:Depends}, + ${shlibs:Depends} Description: MariaDB database common files (e.g. /etc/mysql/conf.d/mariadb.cnf) MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -323,7 +339,9 @@ Provides: default-mysql-client, mysql-client-5.6, mysql-client-5.7, virtual-mysql-client -Recommends: libdbd-mysql-perl (>= 1.2202), libdbi-perl, libterm-readkey-perl +Recommends: libdbd-mysql-perl (>= 1.2202), + libdbi-perl, + libterm-readkey-perl Description: MariaDB database client binaries MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -392,9 +410,14 @@ Description: MariaDB database core server files Package: mariadb-server-10.3 Architecture: any -Suggests: mailx, mariadb-test, netcat-openbsd, tinyca +Suggests: mailx, + mariadb-test, + netcat-openbsd, + tinyca Recommends: libhtml-template-perl -Pre-Depends: adduser (>= 3.40), debconf, mariadb-common (>= ${source:Version}) +Pre-Depends: adduser (>= 3.40), + debconf, + mariadb-common (>= ${source:Version}) Depends: bsdutils, coreutils, findutils, @@ -455,7 +478,8 @@ Replaces: libmariadbclient-dev (<< 5.5.0), mysql-server-5.6, mysql-server-5.7, virtual-mysql-server -Provides: default-mysql-server, virtual-mysql-server +Provides: default-mysql-server, + virtual-mysql-server Description: MariaDB database server binaries MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -466,7 +490,8 @@ Description: MariaDB database server binaries Package: mariadb-server Architecture: all -Depends: mariadb-server-10.3 (>= ${source:Version}), ${misc:Depends} +Depends: mariadb-server-10.3 (>= ${source:Version}), + ${misc:Depends} Description: MariaDB database server (metapackage depending on the latest version) This is an empty package that depends on the current "best" version of mariadb-server (currently mariadb-server-10.3), as determined by the MariaDB @@ -481,7 +506,8 @@ Description: MariaDB database server (metapackage depending on the latest versio Package: mariadb-client Architecture: all -Depends: mariadb-client-10.3 (>= ${source:Version}), ${misc:Depends} +Depends: mariadb-client-10.3 (>= ${source:Version}), + ${misc:Depends} Description: MariaDB database client (metapackage depending on the latest version) This is an empty package that depends on the current "best" version of mariadb-client (currently mariadb-client-10.3), as determined by the MariaDB @@ -650,7 +676,7 @@ Architecture: any Breaks: mariadb-backup-10.1, mariadb-backup-10.2 Replaces: mariadb-backup-10.1, - mariadb-backup-10.2 + mariadb-backup-10.2 Depends: mariadb-server-10.3, ${misc:Depends}, ${shlibs:Depends} @@ -673,8 +699,8 @@ Breaks: mariadb-aws-key-management-10.1, mariadb-aws-key-management-10.2 Replaces: mariadb-aws-key-management-10.1, mariadb-aws-key-management-10.2 -Depends: mariadb-server-10.3, - libcurl3, +Depends: libcurl3, + mariadb-server-10.3, ${misc:Depends}, ${shlibs:Depends} Description: Amazon Web Service Key Management Service Plugin for MariaDB diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index f33febf0e50..72798dec379 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,9 +1,9 @@ usr/bin/mariadb_config usr/include/mariadb usr/lib/*/libmariadb.so -usr/lib/*/libmariadbclient.so usr/lib/*/libmariadbclient.a +usr/lib/*/libmariadbclient.so usr/lib/*/libmysqlservices.a usr/share/aclocal/mysql.m4 -usr/share/pkgconfig/mariadb.pc usr/share/man/man1/mysql_config.1 +usr/share/pkgconfig/mariadb.pc diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install index 68d763ae404..8636166a493 100644 --- a/debian/libmariadb3.install +++ b/debian/libmariadb3.install @@ -1,5 +1,5 @@ usr/lib/*/libmariadb.so.* +usr/lib/mysql/plugin/client_ed25519.so usr/lib/mysql/plugin/dialog.so usr/lib/mysql/plugin/mysql_clear_password.so -usr/lib/mysql/plugin/client_ed25519.so usr/lib/mysql/plugin/sha256_password.so diff --git a/debian/mariadb-client-10.3.docs b/debian/mariadb-client-10.3.docs index 8117d689011..c09092629c3 100644 --- a/debian/mariadb-client-10.3.docs +++ b/debian/mariadb-client-10.3.docs @@ -1,2 +1,2 @@ -debian/additions/innotop/changelog.innotop README.md +debian/additions/innotop/changelog.innotop diff --git a/debian/mariadb-client-10.3.install b/debian/mariadb-client-10.3.install index 75ee09db783..945bf77c689 100644 --- a/debian/mariadb-client-10.3.install +++ b/debian/mariadb-client-10.3.install @@ -10,13 +10,13 @@ usr/bin/mysqldumpslow usr/bin/mysqlimport usr/bin/mysqlshow usr/bin/mysqlslap +usr/share/man/man1/mysql_find_rows.1 +usr/share/man/man1/mysql_fix_extensions.1 +usr/share/man/man1/mysql_waitpid.1 usr/share/man/man1/mysqlaccess.1 usr/share/man/man1/mysqladmin.1 usr/share/man/man1/mysqldump.1 usr/share/man/man1/mysqldumpslow.1 -usr/share/man/man1/mysql_find_rows.1 -usr/share/man/man1/mysql_fix_extensions.1 usr/share/man/man1/mysqlimport.1 usr/share/man/man1/mysqlshow.1 usr/share/man/man1/mysqlslap.1 -usr/share/man/man1/mysql_waitpid.1 diff --git a/debian/mariadb-client-10.3.links b/debian/mariadb-client-10.3.links index 0b86e87f2e9..4a504969246 100644 --- a/debian/mariadb-client-10.3.links +++ b/debian/mariadb-client-10.3.links @@ -1,6 +1,6 @@ -usr/bin/mysqlcheck usr/bin/mysqlrepair usr/bin/mysqlcheck usr/bin/mysqlanalyze usr/bin/mysqlcheck usr/bin/mysqloptimize -usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlrepair.1.gz +usr/bin/mysqlcheck usr/bin/mysqlrepair usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlanalyze.1.gz usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqloptimize.1.gz +usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlrepair.1.gz diff --git a/debian/mariadb-plugin-aws-key-management.install b/debian/mariadb-plugin-aws-key-management.install index 655735cb8ac..199d4c87950 100644 --- a/debian/mariadb-plugin-aws-key-management.install +++ b/debian/mariadb-plugin-aws-key-management.install @@ -1,3 +1,3 @@ -usr/lib/mysql/plugin/aws_key_management.so debian/additions/enable_encryption.preset etc/mysql/mariadb.conf.d etc/mysql/conf.d/aws_key_management.cnf etc/mysql/mariadb.conf.d +usr/lib/mysql/plugin/aws_key_management.so diff --git a/debian/mariadb-plugin-mroonga.install b/debian/mariadb-plugin-mroonga.install index 5cb81419e5e..fedcf62eef0 100644 --- a/debian/mariadb-plugin-mroonga.install +++ b/debian/mariadb-plugin-mroonga.install @@ -1,5 +1,5 @@ usr/lib/mysql/plugin/ha_mroonga.so +usr/share/mysql/mroonga/AUTHORS +usr/share/mysql/mroonga/COPYING usr/share/mysql/mroonga/install.sql usr/share/mysql/mroonga/uninstall.sql -usr/share/mysql/mroonga/COPYING -usr/share/mysql/mroonga/AUTHORS diff --git a/debian/mariadb-plugin-rocksdb.install b/debian/mariadb-plugin-rocksdb.install index ee45a822e0c..224ef5501c5 100644 --- a/debian/mariadb-plugin-rocksdb.install +++ b/debian/mariadb-plugin-rocksdb.install @@ -1,4 +1,4 @@ etc/mysql/conf.d/rocksdb.cnf etc/mysql/mariadb.conf.d -usr/lib/mysql/plugin/ha_rocksdb.so usr/bin/mysql_ldb usr/bin/sst_dump +usr/lib/mysql/plugin/ha_rocksdb.so diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index c0af11c5cc0..d8de5512b05 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -33,15 +33,15 @@ usr/bin/replace usr/bin/resolve_stack_dump usr/bin/resolveip usr/bin/wsrep_sst_common +usr/bin/wsrep_sst_mariabackup usr/bin/wsrep_sst_mysqldump usr/bin/wsrep_sst_rsync usr/bin/wsrep_sst_xtrabackup usr/bin/wsrep_sst_xtrabackup-v2 -usr/bin/wsrep_sst_mariabackup +usr/lib/mysql/plugin/auth_ed25519.so usr/lib/mysql/plugin/auth_pam.so usr/lib/mysql/plugin/auth_socket.so usr/lib/mysql/plugin/file_key_management.so -usr/lib/mysql/plugin/auth_ed25519.so usr/lib/mysql/plugin/ha_archive.so usr/lib/mysql/plugin/ha_blackhole.so usr/lib/mysql/plugin/ha_federated.so @@ -58,14 +58,6 @@ usr/lib/mysql/plugin/sql_errlog.so usr/lib/mysql/plugin/wsrep_info.so usr/share/apport/package-hooks/source_mariadb-10.3.py usr/share/doc/mariadb-server-10.3/mysqld.sym.gz -usr/share/mysql/errmsg-utf8.txt -usr/share/mysql/fill_help_tables.sql -usr/share/mysql/maria_add_gis_sp_bootstrap.sql -usr/share/mysql/mysql_performance_tables.sql -usr/share/mysql/mysql_system_tables.sql -usr/share/mysql/mysql_system_tables_data.sql -usr/share/mysql/mysql_test_data_timezone.sql -usr/share/mysql/wsrep_notify usr/share/man/man1/aria_chk.1 usr/share/man/man1/aria_dump_log.1 usr/share/man/man1/aria_ftdump.1 @@ -75,27 +67,35 @@ usr/share/man/man1/galera_new_cluster.1 usr/share/man/man1/galera_recovery.1 usr/share/man/man1/mariadb-service-convert.1 usr/share/man/man1/msql2mysql.1 -usr/share/man/man1/myisamchk.1 +usr/share/man/man1/my_print_defaults.1 usr/share/man/man1/myisam_ftdump.1 +usr/share/man/man1/myisamchk.1 usr/share/man/man1/myisamlog.1 usr/share/man/man1/myisampack.1 -usr/share/man/man1/my_print_defaults.1 -usr/share/man/man1/mysqlbinlog.1 usr/share/man/man1/mysql_convert_table_format.1 -usr/share/man/man1/mysqld_multi.1 -usr/share/man/man1/mysqld_safe.1 -usr/share/man/man1/mysqlhotcopy.1 usr/share/man/man1/mysql_plugin.1 usr/share/man/man1/mysql_secure_installation.1 usr/share/man/man1/mysql_setpermission.1 usr/share/man/man1/mysql_tzinfo_to_sql.1 +usr/share/man/man1/mysqlbinlog.1 +usr/share/man/man1/mysqld_multi.1 +usr/share/man/man1/mysqld_safe.1 usr/share/man/man1/mysqld_safe_helper.1 +usr/share/man/man1/mysqlhotcopy.1 usr/share/man/man1/perror.1 usr/share/man/man1/replace.1 -usr/share/man/man1/resolveip.1 usr/share/man/man1/resolve_stack_dump.1 +usr/share/man/man1/resolveip.1 usr/share/man/man1/wsrep_sst_common.1 usr/share/man/man1/wsrep_sst_mysqldump.1 usr/share/man/man1/wsrep_sst_rsync.1 usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 usr/share/man/man1/wsrep_sst_xtrabackup.1 +usr/share/mysql/errmsg-utf8.txt +usr/share/mysql/fill_help_tables.sql +usr/share/mysql/maria_add_gis_sp_bootstrap.sql +usr/share/mysql/mysql_performance_tables.sql +usr/share/mysql/mysql_system_tables.sql +usr/share/mysql/mysql_system_tables_data.sql +usr/share/mysql/mysql_test_data_timezone.sql +usr/share/mysql/wsrep_notify diff --git a/debian/mariadb-server-core-10.3.install b/debian/mariadb-server-core-10.3.install index 6244abd1193..46c116b618d 100644 --- a/debian/mariadb-server-core-10.3.install +++ b/debian/mariadb-server-core-10.3.install @@ -2,6 +2,10 @@ usr/bin/innochecksum usr/bin/mysql_install_db usr/bin/mysql_upgrade usr/sbin/mysqld +usr/share/man/man1/innochecksum.1 +usr/share/man/man1/mysql_install_db.1 +usr/share/man/man1/mysql_upgrade.1 +usr/share/man/man8/mysqld.8 usr/share/mysql/charsets usr/share/mysql/czech usr/share/mysql/danish @@ -27,7 +31,3 @@ usr/share/mysql/slovak usr/share/mysql/spanish usr/share/mysql/swedish usr/share/mysql/ukrainian -usr/share/man/man8/mysqld.8 -usr/share/man/man1/innochecksum.1 -usr/share/man/man1/mysql_install_db.1 -usr/share/man/man1/mysql_upgrade.1 diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 4a3ef436634..a3532bde594 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -16,9 +16,16 @@ usr/lib/mysql/plugin/mypluglib.so usr/lib/mysql/plugin/qa_auth_client.so usr/lib/mysql/plugin/qa_auth_interface.so usr/lib/mysql/plugin/qa_auth_server.so +usr/share/man/man1/mysql-stress-test.pl.1 +usr/share/man/man1/mysql-test-run.pl.1 +usr/share/man/man1/mysql_client_test.1 +usr/share/man/man1/mysql_client_test_embedded.1 +usr/share/man/man1/mysqltest.1 +usr/share/man/man1/mysqltest_embedded.1 usr/share/mysql/mysql-test/README usr/share/mysql/mysql-test/README-gcov usr/share/mysql/mysql-test/README.stress +usr/share/mysql/mysql-test/dgcov.pl usr/share/mysql/mysql-test/disabled.def usr/share/mysql/mysql-test/lib usr/share/mysql/mysql-test/mysql-stress-test.pl @@ -26,10 +33,3 @@ usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/purify.supp usr/share/mysql/mysql-test/suite.pm usr/share/mysql/mysql-test/valgrind.supp -usr/share/mysql/mysql-test/dgcov.pl -usr/share/man/man1/mysql-stress-test.pl.1 -usr/share/man/man1/mysql-test-run.pl.1 -usr/share/man/man1/mysql_client_test.1 -usr/share/man/man1/mysql_client_test_embedded.1 -usr/share/man/man1/mysqltest.1 -usr/share/man/man1/mysqltest_embedded.1 diff --git a/debian/mariadb-test.links b/debian/mariadb-test.links index 082680fe5ed..884b25a81da 100644 --- a/debian/mariadb-test.links +++ b/debian/mariadb-test.links @@ -1,2 +1,2 @@ -usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mysql-test-run usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mtr +usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mysql-test-run diff --git a/debian/mysql-common.install b/debian/mysql-common.install index 5ff685f42ff..264df611822 100644 --- a/debian/mysql-common.install +++ b/debian/mysql-common.install @@ -1,2 +1,2 @@ debian/additions/my.cnf etc/mysql -usr/share/mysql-common/internal-use-only \ No newline at end of file +usr/share/mysql-common/internal-use-only From 1d522353c8c2436eea986d00aa155ab00395ba04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 20:32:12 +0000 Subject: [PATCH 80/82] Deb: wrap-and-sort -a the lib* packages in debian/control This way all the libraries are listed in a logical order and it will later be easier to compare the control file contents to downstream versions. --- debian/control | 134 ++++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/debian/control b/debian/control index adc92a58087..9ace4dc8841 100644 --- a/debian/control +++ b/debian/control @@ -38,6 +38,57 @@ Homepage: http://mariadb.org/ Vcs-Git: https://github.com/MariaDB/server.git Vcs-Browser: https://github.com/MariaDB/server/ +Package: libmariadb-dev +Architecture: any +Section: libdevel +Depends: libmariadb3 (= ${binary:Version}), + zlib1g-dev, + ${misc:Depends}, + ${shlibs:Depends} +Breaks: libmariadbclient-dev +Replaces: libmariadbclient-dev +Conflicts: libmariadbclient16-dev +Provides: libmariadbclient-dev +Description: MariaDB database development files + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes development libraries and header files. To allow sources + expecting the MariaDB Connector/C to build. Sources that expect the MySQL + Client libraries should use files from the libmariadb-dev-compat package. + +Package: libmariadb-dev-compat +Architecture: any +Multi-Arch: same +Section: libdevel +Priority: extra +Depends: libmariadb-dev (= ${binary:Version}), + ${misc:Depends} +Conflicts: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev, + libmysqlclient10-dev, + libmysqlclient12-dev, + libmysqlclient14-dev, + libmysqlclient15-dev, + libmysqlclient16-dev +Provides: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev +Replaces: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev +Description: MariaDB Connector/C, compatibility symlinks + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes compatibility symlinks to allow sources expecting the + MySQL client libraries to be built against MariaDB Connector/C. + Package: libmariadb3 Architecture: any Section: libs @@ -114,73 +165,6 @@ Description: Virtual package to satisfy external libmysqlclient18 depends This package provides compatibility symlinks for binaries that expect to find libmysqlclient.so.18 will automatically use libmariadb.so.3 instead. -Package: libmariadb-dev -Architecture: any -Section: libdevel -Depends: libmariadb3 (= ${binary:Version}), - zlib1g-dev, - ${misc:Depends}, - ${shlibs:Depends} -Breaks: libmariadbclient-dev -Replaces: libmariadbclient-dev -Conflicts: libmariadbclient16-dev -Provides: libmariadbclient-dev -Description: MariaDB database development files - MariaDB is a fast, stable and true multi-user, multi-threaded SQL database - server. SQL (Structured Query Language) is the most popular database query - language in the world. The main goals of MariaDB are speed, robustness and - ease of use. - . - This package includes development libraries and header files. To allow sources - expecting the MariaDB Connector/C to build. Sources that expect the MySQL - Client libraries should use files from the libmariadb-dev-compat package. - -Package: libmariadb-dev-compat -Architecture: any -Multi-Arch: same -Section: libdevel -Priority: extra -Depends: libmariadb-dev (= ${binary:Version}), - ${misc:Depends} -Conflicts: libmariadb-client-lgpl-dev-compat, - libmariadbclient-dev-compat, - libmysqlclient-dev, - libmysqlclient10-dev, - libmysqlclient12-dev, - libmysqlclient14-dev, - libmysqlclient15-dev, - libmysqlclient16-dev -Provides: libmariadb-client-lgpl-dev-compat, - libmariadbclient-dev-compat, - libmysqlclient-dev -Replaces: libmariadb-client-lgpl-dev-compat, - libmariadbclient-dev-compat, - libmysqlclient-dev -Description: MariaDB Connector/C, compatibility symlinks - MariaDB is a fast, stable and true multi-user, multi-threaded SQL database - server. SQL (Structured Query Language) is the most popular database query - language in the world. The main goals of MariaDB are speed, robustness and - ease of use. - . - This package includes compatibility symlinks to allow sources expecting the - MySQL client libraries to be built against MariaDB Connector/C. - -Package: libmariadbd19 -Architecture: any -Section: libs -Depends: ${misc:Depends}, - ${shlibs:Depends} -Multi-Arch: same -Breaks: libmariadbd-dev (<< ${source:Version}) -Replaces: libmariadbd-dev (<< ${source:Version}) -Description: MariaDB embedded database, shared library - MariaDB is a fast, stable and true multi-user, multi-threaded SQL database - server. SQL (Structured Query Language) is the most popular database query - language in the world. The main goals of MariaDB are speed, robustness and - ease of use. - . - This package includes a shared library for embedded MariaDB applications - Package: libmariadbd-dev Architecture: any Section: libdevel @@ -200,6 +184,22 @@ Description: MariaDB embedded database, development files . This package includes the embedded server library development and header files. +Package: libmariadbd19 +Architecture: any +Section: libs +Depends: ${misc:Depends}, + ${shlibs:Depends} +Multi-Arch: same +Breaks: libmariadbd-dev (<< ${source:Version}) +Replaces: libmariadbd-dev (<< ${source:Version}) +Description: MariaDB embedded database, shared library + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes a shared library for embedded MariaDB applications + Package: mysql-common Architecture: all Depends: ${misc:Depends}, From 2281fcf38a9f833e053d99564f0cd766e393dba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jan 2018 13:42:33 +0200 Subject: [PATCH 81/82] Follow-up fix to MDEV-14952 for Mariabackup innodb_init_param(): Initialize btr_ahi_parts=1 for Mariabackup. btr_search_enabled: Let the adaptive hash index be disabled in Mariabackup. This would potentially only matter during --export, and --export performs a table scan, not many index lookups. --- extra/mariabackup/xtrabackup.cc | 3 +++ storage/innobase/btr/btr0sea.cc | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 20919cf3a81..d9149321702 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1461,6 +1461,9 @@ innodb_init_param(void) srv_page_size = 0; srv_page_size_shift = 0; +#ifdef BTR_CUR_HASH_ADAPT + btr_ahi_parts = 1; +#endif /* BTR_CUR_HASH_ADAPT */ if (innobase_page_size != (1LL << 14)) { int n_shift = (int)get_bit_shift((ulint) innobase_page_size); diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 1a70dce1c7b..3ca50e00b42 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -45,7 +45,7 @@ Created 2/17/1996 Heikki Tuuri /** Is search system enabled. Search system is protected by array of latches. */ -char btr_search_enabled = true; +char btr_search_enabled; /** Number of adaptive hash index partition. */ ulong btr_ahi_parts; From d87531a6a053fdf8bc828857d9cdc11a97026ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jan 2018 14:05:06 +0200 Subject: [PATCH 82/82] Follow-up to MDEV-14952: Remove some more btr_get_search_latch() Replace some !rw_lock_own() assertions with the stronger !btr_search_own_any(). Remove some redundant btr_get_search_latch() calls. btr_search_update_hash_ref(): Remove a duplicated assertion. btr_search_build_page_hash_index(): Remove a duplicated assertion. rw_lock_s_lock() asserts that the latch is not being held. btr_search_disable_ref_count(): Remove an assertion. The only caller is acquiring all adaptive hash index latches. --- storage/innobase/btr/btr0sea.cc | 32 ++++++++++++----------------- storage/innobase/include/btr0sea.ic | 4 ++-- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 3ca50e00b42..ccd3e2f129f 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -189,13 +189,13 @@ will not guarantee success. @param[in] index index handler */ static void -btr_search_check_free_space_in_heap(dict_index_t* index) +btr_search_check_free_space_in_heap(const dict_index_t* index) { hash_table_t* table; mem_heap_t* heap; - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); table = btr_get_search_table(index); @@ -345,9 +345,6 @@ btr_search_disable_ref_count( for (index = dict_table_get_first_index(table); index != NULL; index = dict_table_get_next_index(index)) { - - ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); - index->search_info->ref_count = 0; } } @@ -464,8 +461,8 @@ btr_search_info_update_hash( ulint n_unique; int cmp; - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (dict_index_is_ibuf(index)) { /* So many deletes are performed on an insert buffer tree @@ -572,14 +569,14 @@ semaphore, to save CPU time! Do not assume the fields are consistent. @param[in,out] block buffer block @param[in] cursor cursor */ static -ibool +bool btr_search_update_block_hash_info( btr_search_t* info, buf_block_t* block, const btr_cur_t* cursor) { - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); ut_ad(rw_lock_own(&block->lock, RW_LOCK_S) || rw_lock_own(&block->lock, RW_LOCK_X)); @@ -626,11 +623,11 @@ btr_search_update_block_hash_info( /* Build a new hash index on the page */ - return(TRUE); + return(true); } } - return(FALSE); + return(false); } /** Updates a hash node reference when it has been unsuccessfully used in a @@ -670,8 +667,8 @@ btr_search_update_hash_ref( } ut_ad(block->page.id.space() == index->space); - ut_a(index == cursor->index); - ut_a(!dict_index_is_ibuf(index)); + ut_ad(index == cursor->index); + ut_ad(!dict_index_is_ibuf(index)); if ((info->n_hash_potential > 0) && (block->curr_n_fields == info->n_fields) @@ -696,7 +693,6 @@ btr_search_update_hash_ref( if (UNIV_LIKELY_NULL(heap)) { mem_heap_free(heap); } - ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); ha_insert_for_fold(btr_get_search_table(index), fold, block, rec); @@ -948,10 +944,9 @@ btr_search_guess_on_hash( } } else { ut_ad(btr_search_enabled); + ut_ad(rw_lock_own(ahi_latch, RW_LOCK_S)); } - ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - rec = (rec_t*) ha_search_and_get_data( btr_get_search_table(index), fold); @@ -1379,7 +1374,6 @@ btr_search_build_page_hash_index( ut_a(!dict_index_is_ibuf(index)); ut_ad(page_is_leaf(block->frame)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); ut_ad(rw_lock_own(&(block->lock), RW_LOCK_S) || rw_lock_own(&(block->lock), RW_LOCK_X)); diff --git a/storage/innobase/include/btr0sea.ic b/storage/innobase/include/btr0sea.ic index 65b953961df..e0052a98639 100644 --- a/storage/innobase/include/btr0sea.ic +++ b/storage/innobase/include/btr0sea.ic @@ -61,8 +61,8 @@ btr_search_info_update( dict_index_t* index, /*!< in: index of the cursor */ btr_cur_t* cursor) /*!< in: cursor which was just positioned */ { - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (dict_index_is_spatial(index) || !btr_search_enabled) { return;