From de13fccfc6b16915190cd740f7fe1e3445dff0e7 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Mon, 11 May 2020 12:50:03 -0400 Subject: [PATCH 01/37] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5048bc29198..bccc9d350b1 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=68 +MYSQL_VERSION_PATCH=69 MYSQL_VERSION_EXTRA= From 76f4ae8295f63e4f3f7b8b72c2120d5da59b302c Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 26 May 2020 01:50:46 +0530 Subject: [PATCH 02/37] MDEV-21495: Conditional jump or move depends on uninitialised value in sel_arg_range_seq_next Initialize the parameter PARAM::max_key_part when we iterate over the ranges to get estimates from EITS. --- mysql-test/r/selectivity.result | 15 +++++++++++++++ mysql-test/r/selectivity_innodb.result | 15 +++++++++++++++ mysql-test/t/selectivity.test | 12 ++++++++++++ sql/opt_range_mrr.cc | 1 + 4 files changed, 43 insertions(+) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index dc7e9494946..9bb197ad056 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -1866,4 +1866,19 @@ id select_type table type possible_keys key key_len ref rows Extra set optimizer_switch= @save_optimizer_switch; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; +# +# MDEV-21495: Conditional jump or move depends on uninitialised value in sel_arg_range_seq_next +# +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 SELECT seq, seq from seq_1_to_100; +set optimizer_use_condition_selectivity=4; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * from t1 WHERE a = 5 and b = 5; +a b +5 5 +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1; # End of 10.1 tests diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index f0cbf1662aa..d0e31374784 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -1876,6 +1876,21 @@ id select_type table type possible_keys key key_len ref rows Extra set optimizer_switch= @save_optimizer_switch; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; +# +# MDEV-21495: Conditional jump or move depends on uninitialised value in sel_arg_range_seq_next +# +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 SELECT seq, seq from seq_1_to_100; +set optimizer_use_condition_selectivity=4; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * from t1 WHERE a = 5 and b = 5; +a b +5 5 +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1; # End of 10.1 tests set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index 7df326edb44..1d0256b1b7e 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -1266,5 +1266,17 @@ set optimizer_switch= @save_optimizer_switch; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; +--echo # +--echo # MDEV-21495: Conditional jump or move depends on uninitialised value in sel_arg_range_seq_next +--echo # + +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 SELECT seq, seq from seq_1_to_100; +set optimizer_use_condition_selectivity=4; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +SELECT * from t1 WHERE a = 5 and b = 5; +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1; + --echo # End of 10.1 tests diff --git a/sql/opt_range_mrr.cc b/sql/opt_range_mrr.cc index 5d62ceda7d4..b1894f4b6c4 100644 --- a/sql/opt_range_mrr.cc +++ b/sql/opt_range_mrr.cc @@ -73,6 +73,7 @@ range_seq_t sel_arg_range_seq_init(void *init_param, uint n_ranges, uint flags) { SEL_ARG_RANGE_SEQ *seq= (SEL_ARG_RANGE_SEQ*)init_param; seq->at_start= TRUE; + seq->param->max_key_part= 0; seq->stack[0].key_tree= NULL; seq->stack[0].min_key= seq->param->min_key; seq->stack[0].min_key_flag= 0; From fbcfbb0e1ce16db6c7debe77119de1141a8c6391 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Tue, 26 May 2020 11:43:43 +0300 Subject: [PATCH 03/37] MDEV-19751 Wrong partitioning by KEY() after primary key dropped Default (empty) field list in partitioning by KEY() clause is assigned from primary key. If primary key is changed the partitioning field list is changed as well, so repartitioning required. Not applicable to any non-primary keys as default field list may be taken only from primary key. --- mysql-test/r/partition_alter.result | 43 +++++++++++++++++++++++++++++ mysql-test/t/partition_alter.test | 39 ++++++++++++++++++++++++++ sql/sql_partition.cc | 22 +++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/mysql-test/r/partition_alter.result b/mysql-test/r/partition_alter.result index 1dced95b17a..8de3d4b45ad 100644 --- a/mysql-test/r/partition_alter.result +++ b/mysql-test/r/partition_alter.result @@ -122,3 +122,46 @@ t1 CREATE TABLE `t1` ( PARTITION `p02` ENGINE = MyISAM, PARTITION `p03` ENGINE = MyISAM) drop table t1; +# +# MDEV-19751 Wrong partitioning by KEY() after key dropped +# +create or replace table t1 (pk int, x timestamp(6), primary key (pk, x)) engine innodb +partition by key() partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +# Inplace for DROP PRIMARY KEY when partitioned by default field list is denied +alter table t1 drop primary key, drop column x, add primary key (pk), algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY +alter table t1 drop primary key, drop column x, add primary key (pk); +select * from t1 partition (p0); +pk +1 +drop table t1; +create or replace table t1 (pk int not null, x timestamp(6), unique u(pk, x)) engine innodb +partition by key() partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +# Same for NOT NULL UNIQUE KEY as this is actually primary key +alter table t1 drop key u, drop column x, add unique (pk), algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY +alter table t1 drop key u, drop column x, add unique (pk); +select * from t1 partition (p0); +pk +1 +drop table t1; +create or replace table t1 (pk int, x timestamp(6), primary key (pk)) engine innodb +partition by key(pk) partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +# Inplace for DROP PRIMARY KEY when partitioned by explicit field list is allowed +alter table t1 drop primary key, add primary key (pk, x), algorithm=inplace; +select * from t1 partition (p0); +pk x +1 2000-01-01 00:00:00.000000 +drop table t1; +create or replace table t1 (k int, x timestamp(6), unique key u (x, k)) engine innodb +partition by key(k) partitions 2; +insert into t1 (k, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +# Inplace for DROP KEY is allowed +alter table t1 drop key u, algorithm=inplace; +select * from t1 partition (p0); +k x +1 2000-01-01 00:00:00.000000 +drop table t1; diff --git a/mysql-test/t/partition_alter.test b/mysql-test/t/partition_alter.test index 1235e5b8dc2..3c783ebe375 100644 --- a/mysql-test/t/partition_alter.test +++ b/mysql-test/t/partition_alter.test @@ -112,3 +112,42 @@ insert into t1 values(0, 1, 1, NULL, now(), now()); alter online table t1 delay_key_write=1; show create table t1; drop table t1; + +--echo # +--echo # MDEV-19751 Wrong partitioning by KEY() after key dropped +--echo # +create or replace table t1 (pk int, x timestamp(6), primary key (pk, x)) engine innodb +partition by key() partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +--echo # Inplace for DROP PRIMARY KEY when partitioned by default field list is denied +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t1 drop primary key, drop column x, add primary key (pk), algorithm=inplace; +alter table t1 drop primary key, drop column x, add primary key (pk); +select * from t1 partition (p0); +drop table t1; + +create or replace table t1 (pk int not null, x timestamp(6), unique u(pk, x)) engine innodb +partition by key() partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +--echo # Same for NOT NULL UNIQUE KEY as this is actually primary key +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t1 drop key u, drop column x, add unique (pk), algorithm=inplace; +alter table t1 drop key u, drop column x, add unique (pk); +select * from t1 partition (p0); +drop table t1; + +create or replace table t1 (pk int, x timestamp(6), primary key (pk)) engine innodb +partition by key(pk) partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +--echo # Inplace for DROP PRIMARY KEY when partitioned by explicit field list is allowed +alter table t1 drop primary key, add primary key (pk, x), algorithm=inplace; +select * from t1 partition (p0); +drop table t1; + +create or replace table t1 (k int, x timestamp(6), unique key u (x, k)) engine innodb +partition by key(k) partitions 2; +insert into t1 (k, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +--echo # Inplace for DROP KEY is allowed +alter table t1 drop key u, algorithm=inplace; +select * from t1 partition (p0); +drop table t1; diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index 071c9c05129..2334286b039 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -5473,6 +5473,28 @@ the generated partition syntax in a correct manner. *partition_changed= TRUE; } } + /* + Prohibit inplace when partitioned by primary key and the primary key is dropped. + */ + if (!*partition_changed && + tab_part_info->part_field_array && + !tab_part_info->part_field_list.elements && + table->s->primary_key != MAX_KEY) + { + KEY *primary_key= table->key_info + table->s->primary_key; + List_iterator_fast drop_it(alter_info->drop_list); + const char *primary_name= primary_key->name; + const Alter_drop *drop; + drop_it.rewind(); + while ((drop= drop_it++)) + { + if (drop->type == Alter_drop::KEY && + 0 == my_strcasecmp(system_charset_info, primary_name, drop->name)) + break; + } + if (drop) + *partition_changed= TRUE; + } } if (thd->work_part_info) { From 9bbd685e8d325c5868a93083addc7a930c1c90de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 26 May 2020 12:23:20 +0300 Subject: [PATCH 04/37] MDEV-22513 main.processlist_notembedded Timeout in wait_until_count_sessions Do not blindly disconnect the connection that is in WAIT_FOR because it could happen that neither the disconnect nor the SIGNAL would be processed before RESET would discard the signal. --- mysql-test/r/processlist_notembedded.result | 1 + mysql-test/t/processlist_notembedded.test | 3 +++ 2 files changed, 4 insertions(+) diff --git a/mysql-test/r/processlist_notembedded.result b/mysql-test/r/processlist_notembedded.result index ab1ced79e25..ed8509ca13c 100644 --- a/mysql-test/r/processlist_notembedded.result +++ b/mysql-test/r/processlist_notembedded.result @@ -4,5 +4,6 @@ SET DEBUG_SYNC= 'before_join_optimize SIGNAL in_sync WAIT_FOR go'; SET DEBUG_SYNC= 'now WAIT_FOR in_sync'; SET DEBUG_SYNC= 'now SIGNAL go'; +user SET DEBUG_SYNC = 'RESET'; End of 5.5 tests diff --git a/mysql-test/t/processlist_notembedded.test b/mysql-test/t/processlist_notembedded.test index b07ac00a44a..90750b40e62 100644 --- a/mysql-test/t/processlist_notembedded.test +++ b/mysql-test/t/processlist_notembedded.test @@ -30,7 +30,10 @@ remove_file $MYSQLTEST_VARDIR/tmp//MDEV-20466.text; SET DEBUG_SYNC= 'now SIGNAL go'; +connection con1; +reap; disconnect con1; +connection default; SET DEBUG_SYNC = 'RESET'; From f1f14c209205b3a7dc375abf0cad19107221d499 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Tue, 26 May 2020 13:14:47 +0300 Subject: [PATCH 05/37] MDEV-20015 Assertion `!in_use->is_error()' failed in TABLE::update_virtual_field update_virtual_field() is called as part of index rebuild in ha_myisam::repair() (MDEV-5800) which is done on bulk INSERT finish. Assertion in update_virtual_field() was put as part of MDEV-16222 because update_virtual_field() returns in_use->is_error(). The idea: wrongly mixed semantics of error status before update_virtual_field() and the status returned by update_virtual_field(). The former can falsely influence the latter. --- mysql-test/suite/vcol/r/vcol_keys_myisam.result | 9 +++++++++ mysql-test/suite/vcol/t/vcol_keys_myisam.test | 11 +++++++++++ sql/table.cc | 8 +++++--- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/vcol/r/vcol_keys_myisam.result b/mysql-test/suite/vcol/r/vcol_keys_myisam.result index b7086600ab1..dd6e4c414f0 100644 --- a/mysql-test/suite/vcol/r/vcol_keys_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_keys_myisam.result @@ -408,3 +408,12 @@ SELECT * FROM t1 WHERE d2 < d1; i d1 d2 t 1 2023-03-16 2023-03-15 1 DROP TABLE t1; +# +# MDEV-20015 Assertion `!in_use->is_error()' failed in TABLE::update_virtual_field +# +create or replace table t1 (a int); +insert into t1 (a) values (1), (1); +create or replace table t2 (pk int, b int, c int as (b) virtual, primary key (pk), key(c)); +insert into t2 (pk) select a from t1; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +drop tables t1, t2; diff --git a/mysql-test/suite/vcol/t/vcol_keys_myisam.test b/mysql-test/suite/vcol/t/vcol_keys_myisam.test index 99b1c9a444b..24612f4d55f 100644 --- a/mysql-test/suite/vcol/t/vcol_keys_myisam.test +++ b/mysql-test/suite/vcol/t/vcol_keys_myisam.test @@ -300,3 +300,14 @@ DROP TABLE t1; # Cleanup --let $datadir= `SELECT @@datadir` --remove_file $datadir/test/load_t1 + + +--echo # +--echo # MDEV-20015 Assertion `!in_use->is_error()' failed in TABLE::update_virtual_field +--echo # +create or replace table t1 (a int); +insert into t1 (a) values (1), (1); +create or replace table t2 (pk int, b int, c int as (b) virtual, primary key (pk), key(c)); +--error ER_DUP_ENTRY +insert into t2 (pk) select a from t1; +drop tables t1, t2; diff --git a/sql/table.cc b/sql/table.cc index 3bb321d3306..d2a45032dbb 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -7778,15 +7778,17 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) int TABLE::update_virtual_field(Field *vf) { - DBUG_ASSERT(!in_use->is_error()); - Query_arena backup_arena; DBUG_ENTER("TABLE::update_virtual_field"); + Query_arena backup_arena; + Counting_error_handler count_errors; + in_use->push_internal_handler(&count_errors); in_use->set_n_backup_active_arena(expr_arena, &backup_arena); bitmap_clear_all(&tmp_set); vf->vcol_info->expr->walk(&Item::update_vcol_processor, 0, &tmp_set); vf->vcol_info->expr->save_in_field(vf, 0); in_use->restore_active_arena(expr_arena, &backup_arena); - DBUG_RETURN(in_use->is_error()); + in_use->pop_internal_handler(); + DBUG_RETURN(count_errors.errors); } From dbe447a78908214614db53061dccbc6bde52764e Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Tue, 5 May 2020 20:32:32 +0300 Subject: [PATCH 06/37] MDEV-15152 Optimistic parallel slave doesnt cope well with START SLAVE UNTIL The immediate bug was caused by a failure to recognize a correct position to stop the slave applier run in optimistic parallel mode. There were the following set of issues that the analysis unveil. 1 incorrect estimate for the event binlog position passed to is_until_satisfied 2 wait for workers to complete by the driver thread did not account non-group events that could be left unprocessed and thus to mix up the last executed binlog group's file and position: the file remained old and the position related to the new rotated file 3 incorrect 'slave reached file:pos' by the parallel slave report in the error log 4 relay log UNTIL missed out the parallel slave branch in is_until_satisfied. The patch addresses all of them to simplify logics of log change notification in either the master and relay-log until case. P.1 is addressed with passing the event into is_until_satisfied() for proper analisis by the function. P.2 is fixed by changes in handle_queued_pos_update(). P.4 required removing relay-log change notification by workers. Instead the driver thread updates the notion of the current relay-log fully itself with aid of introduced bool Relay_log_info::until_relay_log_names_defer. An extra print out of the requested until file:pos is arranged with --log-warning=3. --- .../r/rpl_parallel_optimistic_until.result | 238 +++++++++ .../rpl/t/rpl_parallel_optimistic_until.test | 466 ++++++++++++++++++ sql/rpl_parallel.cc | 6 +- sql/rpl_rli.cc | 65 ++- sql/rpl_rli.h | 19 +- sql/slave.cc | 31 +- 6 files changed, 806 insertions(+), 19 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result create mode 100644 mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test diff --git a/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result b/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result new file mode 100644 index 00000000000..d3d1a52d5db --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result @@ -0,0 +1,238 @@ +include/master-slave.inc +[connection master] +include/stop_slave.inc +RESET MASTER; +RESET SLAVE; +RESET MASTER; +CREATE TABLE t1 (a int primary key, b text) ENGINE=InnoDB; +INSERT INTO t1 SET a=25, b='trx0'; +include/start_slave.inc +include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=2; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; +SET @old_max_relay_log_size = @@global.max_relay_log_size; +SET @@global.max_relay_log_size=4096; +BEGIN; +INSERT INTO t1 SET a=1, b='trx1'; +INSERT INTO t1 SET a=2, b='trx1'; +INSERT INTO t1 SET a=3, b='trx1'; +INSERT INTO t1 SET a=4, b='trx1'; +INSERT INTO t1 SET a=5, b='trx1'; +INSERT INTO t1 SET a=6, b='trx1'; +INSERT INTO t1 SET a=7, b='trx1'; +INSERT INTO t1 SET a=8, b='trx1'; +INSERT INTO t1 SET a=9, b='trx1'; +INSERT INTO t1 SET a=10, b='trx1'; +INSERT INTO t1 SET a=11, b='trx1'; +INSERT INTO t1 SET a=12, b='trx1'; +INSERT INTO t1 SET a=13, b='trx1'; +INSERT INTO t1 SET a=14, b='trx1'; +INSERT INTO t1 SET a=15, b='trx1'; +INSERT INTO t1 SET a=16, b='trx1'; +INSERT INTO t1 SET a=17, b='trx1'; +INSERT INTO t1 SET a=18, b='trx1'; +INSERT INTO t1 SET a=19, b='trx1'; +INSERT INTO t1 SET a=20, b='trx1'; +INSERT INTO t1 SET a=21, b='trx1'; +INSERT INTO t1 SET a=22, b='trx1'; +INSERT INTO t1 SET a=23, b='trx1'; +INSERT INTO t1 SET a=24, b='trx1'; +COMMIT; +FLUSH LOGS; +BEGIN; +UPDATE t1 SET b='trx2_0' WHERE a = 25; +UPDATE t1 SET b='trx2' WHERE a = 25; +COMMIT; +INSERT INTO t1 SET a=26,b='trx3'; +*** case 1 UNTIL inside trx2 +BEGIN; +INSERT INTO t1 SET a= 1; +SELECT <= AND < as "pos_until < trx0 and is within trx2"; +pos_until < trx0 and is within trx2 +1 +CHANGE MASTER TO MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +ROLLBACK; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2'; +trx2 is committed +1 +SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3'; +trx3 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +*** case 2 UNTIL inside trx2 +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +BEGIN; +INSERT INTO t1 SET a= 1; +include/stop_slave.inc +SELECT <= AND < as "pos_until >= trx0 and is within trx2"; +pos_until >= trx0 and is within trx2 +1 +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +ROLLBACK; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2'; +trx2 is committed +1 +SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3'; +trx3 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +*** case 3 UNTIL inside trx1 +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +BEGIN; +INSERT INTO t1 SET a= 1; # block trx1; +include/stop_slave.inc +SELECT < as "pos_until before trx2 start position"; +pos_until before trx2 start position +1 +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +ROLLBACK; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 25-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1'; +trx1 is committed +1 +SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2'; +trx2 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +*** case 4 Relay-log UNTIL inside trx1 +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +BEGIN; +INSERT INTO t1 SET a= 1; # block trx1; +include/stop_slave.inc +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +ROLLBACK; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 25-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1'; +trx1 is committed +1 +SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2'; +trx2 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +*** case 5 Relay-log UNTIL inside a "big" trx that spawns few relay logs +CREATE TABLE t2 (a TEXT) ENGINE=InnoDB; +FLUSH LOGS; +include/stop_slave.inc +BEGIN; +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +COMMIT; +INSERT INTO t2 SET a='a'; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +Proof 2: Resume works out +include/start_slave.inc +include/diff_tables.inc [master:t2,slave:t2] +*** case 6 Relay-log UNTIL inside a small trx inside a sequence of relay logs +include/stop_slave.inc +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +COMMIT; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +include/sync_slave_io_with_master.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +Proof 2: Resume works out +include/start_slave.inc +include/diff_tables.inc [master:t2,slave:t2] +include/stop_slave.inc +SET GLOBAL max_relay_log_size=@old_max_relay_log_size; +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +include/start_slave.inc +DROP TABLE t1, t2; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test b/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test new file mode 100644 index 00000000000..9e42fae8464 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test @@ -0,0 +1,466 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/master-slave.inc +# Format is restricted because the test expects a specific result of +# relay-logging that splits a transaction into two different files. +--source include/have_binlog_format_row.inc + +# +# MDEV-15152 Optimistic parallel slave doesn't cope well with START SLAVE UNTIL +# +--connection slave +--source include/stop_slave.inc +RESET MASTER; +RESET SLAVE; + +--connection master +RESET MASTER; +CREATE TABLE t1 (a int primary key, b text) ENGINE=InnoDB; +--let $a0 = 25 +--eval INSERT INTO t1 SET a=$a0, b='trx0' +# Memorize the position for replication restart from it +--let $pos_trx0 = query_get_value(SHOW MASTER STATUS, Position, 1) + +--connection slave +--source include/start_slave.inc + +--connection master +# --connection slave +--sync_slave_with_master +--source include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=2; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; + +# Run the slave in the following modes one by one. +# +# 1. the until position is set in the middle of trx2 +# below $pos_trx0 of the last exec position in the first file +# 2. and above $pos_trx0 +# In either case trx2 must commit before slave stops. +# 3. the until postion is inside trx1 +# 4. RELAY log until inside trx1 +# 5. RELAY log until inside a "big" trx +# 6. RELAY log until inside a trx within a sequence of relay logs +# +# Execution flaw for Until_Master_Pos cases follows as: +# create the transaction trx1, trx2 +# logged at the beginning of two subsequent binlog files. +# Set the until position to at the middle of the 2rd transaction. +# Engage the optimistic scheduler while having trx1 execution blocked. +# Lift the block after trx2 has reached waiting its order to commit. +# *Proof 1* +# Observe that the slave applier stops at a correct position. +# In the bug condition it would stop prematurely having the stop position +# in the first file, therefore trx2 not committed. +# Specifically, an internal transaction position until makes the applier to run +# beyond it to commit commit the current transaction. +# *Proof 2* +# Observe the following START SLAVE resumes OK. +# +# Auxiliary third trx3 on master is just for triggering the actual stop +# (whihc is a legacy UNTIL's property). +# trx0 is to produce a specific value of the last executed binlog file:pos +# to emulate the bug condition. +# +# Intermediate checks via SELECT are supposed to succeed +# with putting out value 1. +# +# NOTE: Relay log until tests have to use explicit log names and position +# which may require to adjust with future changes to event formats etc. +# + +--connection slave +SET @old_max_relay_log_size = @@global.max_relay_log_size; +SET @@global.max_relay_log_size=4096; + +--connection master +# trx1 +--let $a=1 +BEGIN; +while (`SELECT $a < $a0`) +{ + --eval INSERT INTO t1 SET a=$a, b='trx1' +--inc $a +} +COMMIT; +--let $fil_1 = query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx1 = query_get_value(SHOW MASTER STATUS, Position, 1) + +FLUSH LOGS; + +# $pos_0 the offset of the first event of trx2 in new file +--let $pos_0=query_get_value(SHOW MASTER STATUS, Position, 1) +# trx2 +--let $a=$a0 +BEGIN; +--eval UPDATE t1 SET b='trx2_0' WHERE a = $a +--eval UPDATE t1 SET b='trx2' WHERE a = $a +COMMIT; +--let $fil_2=query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx2=query_get_value(SHOW MASTER STATUS, Position, 1) + +# trx3 +--let $a=$a0 +--inc $a +--eval INSERT INTO t1 SET a=$a,b='trx3' +--let $pos_trx3=query_get_value(SHOW MASTER STATUS, Position, 1) +--let $a= + + +--echo *** case 1 UNTIL inside trx2 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1 +--connection slave +--let $pos_until=`SELECT $pos_trx0 - 1` +--replace_result $pos_0 $pos_until $pos_trx2 +--eval SELECT $pos_0 <= $pos_until AND $pos_until < $pos_trx2 as "pos_until < trx0 and is within trx2" +CHANGE MASTER TO MASTER_USE_GTID=no; +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2' +--eval SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 2 UNTIL inside trx2 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1 + +--connection slave +--source include/stop_slave.inc + +--let $pos_until=`SELECT $pos_trx2 - 1` +--replace_result $pos_trx0 $pos_until $pos_trx2 +--eval SELECT $pos_trx0 <= $pos_until AND $pos_until < $pos_trx2 as "pos_until >= trx0 and is within trx2" +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2' +--eval SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 3 UNTIL inside trx1 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1; # block trx1 + +--connection slave +--source include/stop_slave.inc + +--let $pos_until=`SELECT $pos_0 - 1` +--replace_result $pos_0 $pos_until $pos_trx2 +--eval SELECT $pos_until < $pos_0 as "pos_until before trx2 start position" +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = $a0-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1' +--eval SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 4 Relay-log UNTIL inside trx1 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1; # block trx1 + +--connection slave +--source include/stop_slave.inc +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +# The following test sets the stop coordinate is set to inside the first event +# of a relay log that holds events of a transaction started in an earlier log. +# Peek the stop position in the middle of trx1 (Xid event is here) +# whose last event follow it +--let $pos_until=2568 +--let $file_rl=slave-relay-bin.000002 +--let $binlog_file=$file_rl +--let $info= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_until LIMIT 1, Info, 1) + +if (`SELECT "$info" NOT LIKE "COMMIT /* xid=% */"`) +{ + --echo *** Unexpected offset. Refine it to point to the correct XID event! + --die +} + +# Complicate the test to screw the coordinate away from event boundary. +--dec $pos_until +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +if (`SELECT strcmp("$file_rl","$file_stop") > -1`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $file_rl:$pos_until. + --die +} + +--eval SELECT count(*) = $a0-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1' +--eval SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + + +--echo *** case 5 Relay-log UNTIL inside a "big" trx that spawns few relay logs + +--connection master +CREATE TABLE t2 (a TEXT) ENGINE=InnoDB; +FLUSH LOGS; + +--sync_slave_with_master +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +--let $records=`SELECT floor(4*@@global.max_relay_log_size / 1024) + 1` + +--connection slave +--source include/stop_slave.inc + +--connection master +# trx4 +BEGIN; +--let $i=$records +while ($i) +{ + INSERT INTO t2 SET a=repeat('a',1024); + +--dec $i +} +COMMIT; + +# slave will stop there: +--let $file_trx4 = query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx4 = query_get_value(SHOW MASTER STATUS, Position, 1) + +# trx5 +INSERT INTO t2 SET a='a'; +--let $pos_trx5 = query_get_value(SHOW MASTER STATUS, Position, 1) + +--connection slave +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +# Set position inside the transaction though the value +# specified is beyond that relay log file. +# The coordianate may point to in a different event in future changes +# but should not move away from inside this big group of events. +# So we don't test which event in the transaction it points to. +--let $pos_until= 4500 +--let $file_rl= slave-relay-bin.000010 + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +# It's showed the actual stop occurred before trx5 +if (`SELECT strcmp("$file_trx4", "$file_stop") <> 0 OR $pos_stop >= $pos_trx5 OR count(*) <> $records FROM t2`) +{ + --echo *** ERROR: Slave stopped at *binlog* $file_stop:$pos_stop which is not $file_trx4:$pos_trx4. + --die +} + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + +--let $diff_tables=master:t2,slave:t2 +--source include/diff_tables.inc + + + +--echo *** case 6 Relay-log UNTIL inside a small trx inside a sequence of relay logs + +--connection slave +--source include/stop_slave.inc + +--connection master +# trx6 +--let $records=`SELECT count(*) FROM t2` +while ($records) +{ + BEGIN; + DELETE FROM t2 LIMIT 1; + COMMIT; +--dec $records +} +COMMIT; + +--connection slave +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +--connection master +--source include/sync_slave_io_with_master.inc + +--connection slave +# The relay-log coordinate is not at an event boundary and +# also may change across the server version. +# The test makes its best to check its coherance. +--let $pos_until= 4100 +--let $file_rl= slave-relay-bin.000015 + +--let $pos_gtid = 4010 +--let $info= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_gtid LIMIT 1, Info, 1) + +if (`SELECT "$info" != "BEGIN GTID 0-1-23"`) +{ + --echo *** Unexpected offset. Refine it to point to the correct GTID! + --die +} +--let $pos_event = 4090 +--let $type= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_event LIMIT 1, Event_type, 1) +if (`SELECT "$type" != "Delete_rows_v1"`) +{ + --echo *** Unexpected offset. Refine it to point to the expected event! + --die +} + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +if (`SELECT strcmp("$file_stop", "$file_rl") = -1 OR + strcmp("$file_stop", "$file_rl") = 0 AND $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at *relay* $file_stop:$pos_stop which is not $file_rl:$pos_until. + --die +} + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + +--let $diff_tables=master:t2,slave:t2 +--source include/diff_tables.inc + +# +# Clean up. +# +--connection slave +--source include/stop_slave.inc +SET GLOBAL max_relay_log_size=@old_max_relay_log_size; +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +--source include/start_slave.inc + +--connection master +DROP TABLE t1, t2; + +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 574730b96c1..fb6f23af295 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -75,18 +75,18 @@ handle_queued_pos_update(THD *thd, rpl_parallel_thread::queued_event *qev) /* Do not update position if an earlier event group caused an error abort. */ DBUG_ASSERT(qev->typ == rpl_parallel_thread::queued_event::QUEUED_POS_UPDATE); + rli= qev->rgi->rli; e= qev->entry_for_queued; - if (e->stop_on_error_sub_id < (uint64)ULONGLONG_MAX || e->force_abort) + if (e->stop_on_error_sub_id < (uint64)ULONGLONG_MAX || + (e->force_abort && !rli->stop_for_until)) return; - rli= qev->rgi->rli; mysql_mutex_lock(&rli->data_lock); cmp= strcmp(rli->group_relay_log_name, qev->event_relay_log_name); if (cmp < 0) { rli->group_relay_log_pos= qev->future_event_relay_log_pos; strmake_buf(rli->group_relay_log_name, qev->event_relay_log_name); - rli->notify_group_relay_log_name_update(); } else if (cmp == 0 && rli->group_relay_log_pos < qev->future_event_relay_log_pos) rli->group_relay_log_pos= qev->future_event_relay_log_pos; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index f3514253874..905462ecfbd 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -65,6 +65,7 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) gtid_skip_flag(GTID_SKIP_NOT), inited(0), abort_slave(0), stop_for_until(0), slave_running(MYSQL_SLAVE_NOT_RUN), until_condition(UNTIL_NONE), until_log_pos(0), retried_trans(0), executed_entries(0), + until_relay_log_names_defer(false), m_flags(0) { DBUG_ENTER("Relay_log_info::Relay_log_info"); @@ -512,6 +513,8 @@ void Relay_log_info::clear_until_condition() until_condition= Relay_log_info::UNTIL_NONE; until_log_name[0]= 0; until_log_pos= 0; + until_relay_log_names_defer= false; + DBUG_VOID_RETURN; } @@ -997,7 +1000,6 @@ void Relay_log_info::inc_group_relay_log_pos(ulonglong log_pos, { group_relay_log_pos= rgi->future_event_relay_log_pos; strmake_buf(group_relay_log_name, rgi->event_relay_log_name); - notify_group_relay_log_name_update(); } else if (cmp == 0 && group_relay_log_pos < rgi->future_event_relay_log_pos) group_relay_log_pos= rgi->future_event_relay_log_pos; @@ -1258,29 +1260,78 @@ err: autoincrement or if we have transactions). Should be called ONLY if until_condition != UNTIL_NONE ! + + In the parallel execution mode and UNTIL_MASTER_POS the file name is + presented by future_event_master_log_name which may be ahead of + group_master_log_name. Log_event::log_pos does relate to it nevertheless + so the pair comprises a correct binlog coordinate. + Internal group events and events that have zero log_pos also + produce the zero for the local log_pos which may not lead to the + function falsely return true. + In UNTIL_RELAY_POS the original caching and notification are simplified + to straightforward files comparison when the current event can't be + a part of an event group. + RETURN VALUE true - condition met or error happened (condition seems to have bad log file name) false - condition not met */ -bool Relay_log_info::is_until_satisfied(my_off_t master_beg_pos) +bool Relay_log_info::is_until_satisfied(Log_event *ev) { const char *log_name; ulonglong log_pos; + /* Prevents stopping within transaction; needed solely for Relay UNTIL. */ + bool in_trans= false; + DBUG_ENTER("Relay_log_info::is_until_satisfied"); if (until_condition == UNTIL_MASTER_POS) { log_name= (mi->using_parallel() ? future_event_master_log_name : group_master_log_name); - log_pos= master_beg_pos; + log_pos= (get_flag(Relay_log_info::IN_TRANSACTION) || !ev || !ev->log_pos) ? + (mi->using_parallel() ? 0 : group_master_log_pos) : + ev->log_pos - ev->data_written; } else { DBUG_ASSERT(until_condition == UNTIL_RELAY_POS); - log_name= group_relay_log_name; - log_pos= group_relay_log_pos; + if (!mi->using_parallel()) + { + log_name= group_relay_log_name; + log_pos= group_relay_log_pos; + } + else + { + log_name= event_relay_log_name; + log_pos= event_relay_log_pos; + in_trans= get_flag(Relay_log_info::IN_TRANSACTION); + /* + until_log_names_cmp_result is set to UNKNOWN either + - by a non-group event *and* only when it is in the middle of a group + - or by a group event when the preceding group made the above + non-group event to defer the resetting. + */ + if ((ev && !Log_event::is_group_event(ev->get_type_code()))) + { + if (in_trans) + { + until_relay_log_names_defer= true; + } + else + { + until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; + until_relay_log_names_defer= false; + } + } + else if (!in_trans && until_relay_log_names_defer) + { + until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; + until_relay_log_names_defer= false; + } + } } DBUG_PRINT("info", ("group_master_log_name='%s', group_master_log_pos=%llu", @@ -1334,8 +1385,8 @@ bool Relay_log_info::is_until_satisfied(my_off_t master_beg_pos) } DBUG_RETURN(((until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_EQUAL && - log_pos >= until_log_pos) || - until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_GREATER)); + (log_pos >= until_log_pos && !in_trans)) || + until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_GREATER)); } diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 3dec5b0f044..29e47d302bc 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -402,7 +402,7 @@ public: void close_temporary_tables(); /* Check if UNTIL condition is satisfied. See slave.cc for more. */ - bool is_until_satisfied(my_off_t); + bool is_until_satisfied(Log_event *ev); inline ulonglong until_pos() { DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || @@ -410,7 +410,13 @@ public: return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_pos : group_relay_log_pos); } - + inline char *until_name() + { + DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || + until_condition == UNTIL_RELAY_POS); + return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_name : + group_relay_log_name); + } /** Helper function to do after statement completion. @@ -483,6 +489,15 @@ public: } private: + /* + Hint for when to stop event distribution by sql driver thread. + The flag is set ON by a non-group event when this event is in the middle + of a group (e.g a transaction group) so it's too early + to refresh the current-relay-log vs until-log cached comparison result. + And it is checked and to decide whether it's a right time to do so + when the being processed group has been fully scheduled. + */ + bool until_relay_log_names_defer; /* Holds the state of the data in the relay log. diff --git a/sql/slave.cc b/sql/slave.cc index 0ce9c4cbbcd..06f2b0d955a 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3784,12 +3784,8 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && (ev->server_id != global_system_variables.server_id || rli->replicate_same_server_id) && - rli->is_until_satisfied((rli->get_flag(Relay_log_info::IN_TRANSACTION) || !ev->log_pos) - ? rli->group_master_log_pos - : ev->log_pos - ev->data_written)) + rli->is_until_satisfied(ev)) { - sql_print_information("Slave SQL thread stopped because it reached its" - " UNTIL position %llu", rli->until_pos()); /* Setting abort_slave flag because we do not want additional message about error in query execution to be printed. @@ -4943,10 +4939,14 @@ pthread_handler_t handle_slave_sql(void *arg) } if ((rli->until_condition == Relay_log_info::UNTIL_MASTER_POS || rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && - rli->is_until_satisfied(rli->group_master_log_pos)) + rli->is_until_satisfied(NULL)) { sql_print_information("Slave SQL thread stopped because it reached its" - " UNTIL position %llu", rli->until_pos()); + " UNTIL position %llu in %s %s file", + rli->until_pos(), rli->until_name(), + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); mysql_mutex_unlock(&rli->data_lock); goto err; } @@ -5011,7 +5011,24 @@ pthread_handler_t handle_slave_sql(void *arg) err: if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); + /* Gtid_list_log_event::do_apply_event has already reported the GTID until */ + if (rli->stop_for_until && rli->until_condition != Relay_log_info::UNTIL_GTID) + { + if (global_system_variables.log_warnings > 2) + sql_print_information("Slave SQL thread UNTIL stop was requested at position " + "%llu in %s %s file", + rli->until_log_pos, rli->until_log_name, + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); + sql_print_information("Slave SQL thread stopped because it reached its" + " UNTIL position %llu in %s %s file", + rli->until_pos(), rli->until_name(), + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); + }; /* Thread stopped. Print the current replication position to the log */ { StringBuffer<100> tmp; From 0c1f97b3ab7befdae522aaa738be00f6700cee8e Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Tue, 5 May 2020 20:32:32 +0300 Subject: [PATCH 07/37] MDEV-15152 Optimistic parallel slave doesnt cope well with START SLAVE UNTIL The immediate bug was caused by a failure to recognize a correct position to stop the slave applier run in optimistic parallel mode. There were the following set of issues that the analysis unveil. 1 incorrect estimate for the event binlog position passed to is_until_satisfied 2 wait for workers to complete by the driver thread did not account non-group events that could be left unprocessed and thus to mix up the last executed binlog group's file and position: the file remained old and the position related to the new rotated file 3 incorrect 'slave reached file:pos' by the parallel slave report in the error log 4 relay log UNTIL missed out the parallel slave branch in is_until_satisfied. The patch addresses all of them to simplify logics of log change notification in either the master and relay-log until case. P.1 is addressed with passing the event into is_until_satisfied() for proper analisis by the function. P.2 is fixed by changes in handle_queued_pos_update(). P.4 required removing relay-log change notification by workers. Instead the driver thread updates the notion of the current relay-log fully itself with aid of introduced bool Relay_log_info::until_relay_log_names_defer. An extra print out of the requested until file:pos is arranged with --log-warning=3. --- .../r/rpl_parallel_optimistic_until.result | 291 +++++++++++ .../rpl/t/rpl_parallel_optimistic_until.test | 465 ++++++++++++++++++ sql/rpl_parallel.cc | 6 +- sql/rpl_rli.cc | 65 ++- sql/rpl_rli.h | 21 +- sql/slave.cc | 31 +- 6 files changed, 859 insertions(+), 20 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result create mode 100644 mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test diff --git a/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result b/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result new file mode 100644 index 00000000000..a83a9b61b9f --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result @@ -0,0 +1,291 @@ +include/master-slave.inc +[connection master] +connection slave; +include/stop_slave.inc +RESET MASTER; +RESET SLAVE; +connection master; +RESET MASTER; +CREATE TABLE t1 (a int primary key, b text) ENGINE=InnoDB; +INSERT INTO t1 SET a=25, b='trx0'; +connection slave; +include/start_slave.inc +connection master; +connection slave; +include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=2; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; +connection slave; +SET @old_max_relay_log_size = @@global.max_relay_log_size; +SET @@global.max_relay_log_size=4096; +connection master; +BEGIN; +INSERT INTO t1 SET a=1, b='trx1'; +INSERT INTO t1 SET a=2, b='trx1'; +INSERT INTO t1 SET a=3, b='trx1'; +INSERT INTO t1 SET a=4, b='trx1'; +INSERT INTO t1 SET a=5, b='trx1'; +INSERT INTO t1 SET a=6, b='trx1'; +INSERT INTO t1 SET a=7, b='trx1'; +INSERT INTO t1 SET a=8, b='trx1'; +INSERT INTO t1 SET a=9, b='trx1'; +INSERT INTO t1 SET a=10, b='trx1'; +INSERT INTO t1 SET a=11, b='trx1'; +INSERT INTO t1 SET a=12, b='trx1'; +INSERT INTO t1 SET a=13, b='trx1'; +INSERT INTO t1 SET a=14, b='trx1'; +INSERT INTO t1 SET a=15, b='trx1'; +INSERT INTO t1 SET a=16, b='trx1'; +INSERT INTO t1 SET a=17, b='trx1'; +INSERT INTO t1 SET a=18, b='trx1'; +INSERT INTO t1 SET a=19, b='trx1'; +INSERT INTO t1 SET a=20, b='trx1'; +INSERT INTO t1 SET a=21, b='trx1'; +INSERT INTO t1 SET a=22, b='trx1'; +INSERT INTO t1 SET a=23, b='trx1'; +INSERT INTO t1 SET a=24, b='trx1'; +COMMIT; +FLUSH LOGS; +BEGIN; +UPDATE t1 SET b='trx2_0' WHERE a = 25; +UPDATE t1 SET b='trx2' WHERE a = 25; +COMMIT; +INSERT INTO t1 SET a=26,b='trx3'; +*** case 1 UNTIL inside trx2 +connection slave1; +BEGIN; +INSERT INTO t1 SET a= 1; +connection slave; +SELECT <= AND < as "pos_until < trx0 and is within trx2"; +pos_until < trx0 and is within trx2 +1 +CHANGE MASTER TO MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +connection slave1; +ROLLBACK; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2'; +trx2 is committed +1 +SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3'; +trx3 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +*** case 2 UNTIL inside trx2 +connection slave; +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +connection slave1; +BEGIN; +INSERT INTO t1 SET a= 1; +connection slave; +include/stop_slave.inc +SELECT <= AND < as "pos_until >= trx0 and is within trx2"; +pos_until >= trx0 and is within trx2 +1 +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +connection slave1; +ROLLBACK; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2'; +trx2 is committed +1 +SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3'; +trx3 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +*** case 3 UNTIL inside trx1 +connection slave; +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +connection slave1; +BEGIN; +INSERT INTO t1 SET a= 1; # block trx1; +connection slave; +include/stop_slave.inc +SELECT < as "pos_until before trx2 start position"; +pos_until before trx2 start position +1 +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +connection slave1; +ROLLBACK; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 25-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1'; +trx1 is committed +1 +SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2'; +trx2 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +*** case 4 Relay-log UNTIL inside trx1 +connection slave; +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +connection slave1; +BEGIN; +INSERT INTO t1 SET a= 1; # block trx1; +connection slave; +include/stop_slave.inc +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +connection slave1; +ROLLBACK; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 25-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1'; +trx1 is committed +1 +SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2'; +trx2 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +*** case 5 Relay-log UNTIL inside a "big" trx that spawns few relay logs +connection master; +CREATE TABLE t2 (a TEXT) ENGINE=InnoDB; +FLUSH LOGS; +connection slave; +connection slave; +include/stop_slave.inc +connection master; +BEGIN; +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +COMMIT; +INSERT INTO t2 SET a='a'; +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +include/diff_tables.inc [master:t2,slave:t2] +*** case 6 Relay-log UNTIL inside a small trx inside a sequence of relay logs +connection slave; +include/stop_slave.inc +connection master; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +COMMIT; +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +connection master; +include/sync_slave_io_with_master.inc +connection slave; +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +include/diff_tables.inc [master:t2,slave:t2] +connection slave; +include/stop_slave.inc +SET GLOBAL max_relay_log_size=@old_max_relay_log_size; +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +include/start_slave.inc +connection master; +DROP TABLE t1, t2; +connection slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test b/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test new file mode 100644 index 00000000000..508213c9075 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test @@ -0,0 +1,465 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/master-slave.inc +# Format is restricted because the test expects a specific result of +# relay-logging that splits a transaction into two different files. +--source include/have_binlog_format_row.inc + +# +# MDEV-15152 Optimistic parallel slave doesn't cope well with START SLAVE UNTIL +# +--connection slave +--source include/stop_slave.inc +RESET MASTER; +RESET SLAVE; + +--connection master +RESET MASTER; +CREATE TABLE t1 (a int primary key, b text) ENGINE=InnoDB; +--let $a0 = 25 +--eval INSERT INTO t1 SET a=$a0, b='trx0' +# Memorize the position for replication restart from it +--let $pos_trx0 = query_get_value(SHOW MASTER STATUS, Position, 1) + +--connection slave +--source include/start_slave.inc + +--connection master +# --connection slave +--sync_slave_with_master +--source include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=2; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; + +# Run the slave in the following modes one by one. +# +# 1. the until position is set in the middle of trx2 +# below $pos_trx0 of the last exec position in the first file +# 2. and above $pos_trx0 +# In either case trx2 must commit before slave stops. +# 3. the until postion is inside trx1 +# 4. RELAY log until inside trx1 +# 5. RELAY log until inside a "big" trx +# 6. RELAY log until inside a trx within a sequence of relay logs +# +# Execution flaw for Until_Master_Pos cases follows as: +# create the transaction trx1, trx2 +# logged at the beginning of two subsequent binlog files. +# Set the until position to at the middle of the 2rd transaction. +# Engage the optimistic scheduler while having trx1 execution blocked. +# Lift the block after trx2 has reached waiting its order to commit. +# *Proof 1* +# Observe that the slave applier stops at a correct position. +# In the bug condition it would stop prematurely having the stop position +# in the first file, therefore trx2 not committed. +# Specifically, an internal transaction position until makes the applier to run +# beyond it to commit commit the current transaction. +# *Proof 2* +# Observe the following START SLAVE resumes OK. +# +# Auxiliary third trx3 on master is just for triggering the actual stop +# (whihc is a legacy UNTIL's property). +# trx0 is to produce a specific value of the last executed binlog file:pos +# to emulate the bug condition. +# +# Intermediate checks via SELECT are supposed to succeed +# with putting out value 1. +# +# NOTE: Relay log until tests have to use explicit log names and position +# which may require to adjust with future changes to event formats etc. +# + +--connection slave +SET @old_max_relay_log_size = @@global.max_relay_log_size; +SET @@global.max_relay_log_size=4096; + +--connection master +# trx1 +--let $a=1 +BEGIN; +while (`SELECT $a < $a0`) +{ + --eval INSERT INTO t1 SET a=$a, b='trx1' +--inc $a +} +COMMIT; +--let $fil_1 = query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx1 = query_get_value(SHOW MASTER STATUS, Position, 1) + +FLUSH LOGS; + +# $pos_0 the offset of the first event of trx2 in new file +--let $pos_0=query_get_value(SHOW MASTER STATUS, Position, 1) +# trx2 +--let $a=$a0 +BEGIN; +--eval UPDATE t1 SET b='trx2_0' WHERE a = $a +--eval UPDATE t1 SET b='trx2' WHERE a = $a +COMMIT; +--let $fil_2=query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx2=query_get_value(SHOW MASTER STATUS, Position, 1) + +# trx3 +--let $a=$a0 +--inc $a +--eval INSERT INTO t1 SET a=$a,b='trx3' +--let $pos_trx3=query_get_value(SHOW MASTER STATUS, Position, 1) +--let $a= + + +--echo *** case 1 UNTIL inside trx2 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1 +--connection slave +--let $pos_until=`SELECT $pos_trx0 - 1` +--replace_result $pos_0 $pos_until $pos_trx2 +--eval SELECT $pos_0 <= $pos_until AND $pos_until < $pos_trx2 as "pos_until < trx0 and is within trx2" +CHANGE MASTER TO MASTER_USE_GTID=no; +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2' +--eval SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 2 UNTIL inside trx2 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1 + +--connection slave +--source include/stop_slave.inc + +--let $pos_until=`SELECT $pos_trx2 - 1` +--replace_result $pos_trx0 $pos_until $pos_trx2 +--eval SELECT $pos_trx0 <= $pos_until AND $pos_until < $pos_trx2 as "pos_until >= trx0 and is within trx2" +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2' +--eval SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 3 UNTIL inside trx1 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1; # block trx1 + +--connection slave +--source include/stop_slave.inc + +--let $pos_until=`SELECT $pos_0 - 1` +--replace_result $pos_0 $pos_until $pos_trx2 +--eval SELECT $pos_until < $pos_0 as "pos_until before trx2 start position" +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = $a0-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1' +--eval SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 4 Relay-log UNTIL inside trx1 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1; # block trx1 + +--connection slave +--source include/stop_slave.inc +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +# The following test sets the stop coordinate is set to inside the first event +# of a relay log that holds events of a transaction started in an earlier log. +# Peek the stop position in the middle of trx1, not even on a event boundary. +--let $pos_until=255 +--let $file_rl=slave-relay-bin.000003 +--let $binlog_file=$file_rl + +--let $pos_xid=508 +--let $info= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_xid LIMIT 1, Info, 1) + +if (`SELECT "$info" NOT LIKE "COMMIT /* xid=% */" OR $pos_xid < $pos_until`) +{ + --echo *** Unexpected offset. Refine it to point to the correct XID event! + --die +} + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +if (`SELECT strcmp("$file_rl","$file_stop") > -1`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $file_rl:$pos_until. + --die +} + +--eval SELECT count(*) = $a0-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1' +--eval SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + + +--echo *** case 5 Relay-log UNTIL inside a "big" trx that spawns few relay logs + +--connection master +CREATE TABLE t2 (a TEXT) ENGINE=InnoDB; +FLUSH LOGS; + +--sync_slave_with_master +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +--let $records=`SELECT floor(4*@@global.max_relay_log_size / 1024) + 1` + +--connection slave +--source include/stop_slave.inc + +--connection master +# trx4 +BEGIN; +--let $i=$records +while ($i) +{ + INSERT INTO t2 SET a=repeat('a',1024); + +--dec $i +} +COMMIT; + +# slave will stop there: +--let $file_trx4 = query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx4 = query_get_value(SHOW MASTER STATUS, Position, 1) + +# trx5 +INSERT INTO t2 SET a='a'; +--let $pos_trx5 = query_get_value(SHOW MASTER STATUS, Position, 1) + +--connection slave +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +# Set position inside the transaction though the value +# specified is beyond that relay log file. +# The coordianate may point to in a different event in future changes +# but should not move away from inside this big group of events. +# So we don't test which event in the transaction it points to. +--let $pos_until= 4500 +--let $file_rl= slave-relay-bin.000010 + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +# It's showed the actual stop occurred before trx5 +if (`SELECT strcmp("$file_trx4", "$file_stop") <> 0 OR $pos_stop >= $pos_trx5 OR count(*) <> $records FROM t2`) +{ + --echo *** ERROR: Slave stopped at *binlog* $file_stop:$pos_stop which is not $file_trx4:$pos_trx4. + --die +} + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + +--let $diff_tables=master:t2,slave:t2 +--source include/diff_tables.inc + + + +--echo *** case 6 Relay-log UNTIL inside a small trx inside a sequence of relay logs + +--connection slave +--source include/stop_slave.inc + +--connection master +# trx6 +--let $records=`SELECT count(*) FROM t2` +while ($records) +{ + BEGIN; + DELETE FROM t2 LIMIT 1; + COMMIT; +--dec $records +} +COMMIT; + +--connection slave +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +--connection master +--source include/sync_slave_io_with_master.inc + +--connection slave +# The relay-log coordinate is not at an event boundary and +# also may change across the server version. +# The test makes its best to check its coherance. +--let $pos_until= 3130 +--let $file_rl= slave-relay-bin.000018 + +--let $pos_gtid = 2987 +--let $info= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_gtid LIMIT 1, Info, 1) + +if (`SELECT "$info" != "BEGIN GTID 0-1-23"`) +{ + --echo *** Unexpected offset. Refine it to point to the correct GTID! + --die +} +--let $pos_event = 3120 +--let $type= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_event LIMIT 1, Event_type, 1) +if (`SELECT "$type" != "Delete_rows_v1"`) +{ + --echo *** Unexpected offset. Refine it to point to the expected event! + --die +} + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +if (`SELECT strcmp("$file_stop", "$file_rl") = -1 OR + strcmp("$file_stop", "$file_rl") = 0 AND $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at *relay* $file_stop:$pos_stop which is not $file_rl:$pos_until. + --die +} + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + +--let $diff_tables=master:t2,slave:t2 +--source include/diff_tables.inc + +# +# Clean up. +# +--connection slave +--source include/stop_slave.inc +SET GLOBAL max_relay_log_size=@old_max_relay_log_size; +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +--source include/start_slave.inc + +--connection master +DROP TABLE t1, t2; + +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 027aa5f628c..3871ddcf8ef 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -75,18 +75,18 @@ handle_queued_pos_update(THD *thd, rpl_parallel_thread::queued_event *qev) /* Do not update position if an earlier event group caused an error abort. */ DBUG_ASSERT(qev->typ == rpl_parallel_thread::queued_event::QUEUED_POS_UPDATE); + rli= qev->rgi->rli; e= qev->entry_for_queued; - if (e->stop_on_error_sub_id < (uint64)ULONGLONG_MAX || e->force_abort) + if (e->stop_on_error_sub_id < (uint64)ULONGLONG_MAX || + (e->force_abort && !rli->stop_for_until)) return; - rli= qev->rgi->rli; mysql_mutex_lock(&rli->data_lock); cmp= strcmp(rli->group_relay_log_name, qev->event_relay_log_name); if (cmp < 0) { rli->group_relay_log_pos= qev->future_event_relay_log_pos; strmake_buf(rli->group_relay_log_name, qev->event_relay_log_name); - rli->notify_group_relay_log_name_update(); } else if (cmp == 0 && rli->group_relay_log_pos < qev->future_event_relay_log_pos) rli->group_relay_log_pos= qev->future_event_relay_log_pos; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 3e1bb28f701..7215cdd4b96 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -60,6 +60,7 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) slave_running(MYSQL_SLAVE_NOT_RUN), until_condition(UNTIL_NONE), until_log_pos(0), retried_trans(0), executed_entries(0), sql_delay(0), sql_delay_end(0), + until_relay_log_names_defer(false), m_flags(0) { DBUG_ENTER("Relay_log_info::Relay_log_info"); @@ -499,6 +500,8 @@ void Relay_log_info::clear_until_condition() until_condition= Relay_log_info::UNTIL_NONE; until_log_name[0]= 0; until_log_pos= 0; + until_relay_log_names_defer= false; + DBUG_VOID_RETURN; } @@ -989,7 +992,6 @@ void Relay_log_info::inc_group_relay_log_pos(ulonglong log_pos, { group_relay_log_pos= rgi->future_event_relay_log_pos; strmake_buf(group_relay_log_name, rgi->event_relay_log_name); - notify_group_relay_log_name_update(); } else if (cmp == 0 && group_relay_log_pos < rgi->future_event_relay_log_pos) group_relay_log_pos= rgi->future_event_relay_log_pos; @@ -1279,29 +1281,78 @@ err: autoincrement or if we have transactions). Should be called ONLY if until_condition != UNTIL_NONE ! + + In the parallel execution mode and UNTIL_MASTER_POS the file name is + presented by future_event_master_log_name which may be ahead of + group_master_log_name. Log_event::log_pos does relate to it nevertheless + so the pair comprises a correct binlog coordinate. + Internal group events and events that have zero log_pos also + produce the zero for the local log_pos which may not lead to the + function falsely return true. + In UNTIL_RELAY_POS the original caching and notification are simplified + to straightforward files comparison when the current event can't be + a part of an event group. + RETURN VALUE true - condition met or error happened (condition seems to have bad log file name) false - condition not met */ -bool Relay_log_info::is_until_satisfied(my_off_t master_beg_pos) +bool Relay_log_info::is_until_satisfied(Log_event *ev) { const char *log_name; ulonglong log_pos; + /* Prevents stopping within transaction; needed solely for Relay UNTIL. */ + bool in_trans= false; + DBUG_ENTER("Relay_log_info::is_until_satisfied"); if (until_condition == UNTIL_MASTER_POS) { log_name= (mi->using_parallel() ? future_event_master_log_name : group_master_log_name); - log_pos= master_beg_pos; + log_pos= (get_flag(Relay_log_info::IN_TRANSACTION) || !ev || !ev->log_pos) ? + (mi->using_parallel() ? 0 : group_master_log_pos) : + ev->log_pos - ev->data_written; } else { DBUG_ASSERT(until_condition == UNTIL_RELAY_POS); - log_name= group_relay_log_name; - log_pos= group_relay_log_pos; + if (!mi->using_parallel()) + { + log_name= group_relay_log_name; + log_pos= group_relay_log_pos; + } + else + { + log_name= event_relay_log_name; + log_pos= event_relay_log_pos; + in_trans= get_flag(Relay_log_info::IN_TRANSACTION); + /* + until_log_names_cmp_result is set to UNKNOWN either + - by a non-group event *and* only when it is in the middle of a group + - or by a group event when the preceding group made the above + non-group event to defer the resetting. + */ + if ((ev && !Log_event::is_group_event(ev->get_type_code()))) + { + if (in_trans) + { + until_relay_log_names_defer= true; + } + else + { + until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; + until_relay_log_names_defer= false; + } + } + else if (!in_trans && until_relay_log_names_defer) + { + until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; + until_relay_log_names_defer= false; + } + } } DBUG_PRINT("info", ("group_master_log_name='%s', group_master_log_pos=%llu", @@ -1355,8 +1406,8 @@ bool Relay_log_info::is_until_satisfied(my_off_t master_beg_pos) } DBUG_RETURN(((until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_EQUAL && - log_pos >= until_log_pos) || - until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_GREATER)); + (log_pos >= until_log_pos && !in_trans)) || + until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_GREATER)); } diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 4ec4821b67d..2bc0a80268a 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -219,7 +219,7 @@ public: */ char future_event_master_log_name[FN_REFLEN]; - /* + /* Original log name and position of the group we're currently executing (whose coordinates are group_relay_log_name/pos in the relay log) in the master's binlog. These concern the *group*, because in the master's @@ -419,7 +419,7 @@ public: void close_temporary_tables(); /* Check if UNTIL condition is satisfied. See slave.cc for more. */ - bool is_until_satisfied(my_off_t); + bool is_until_satisfied(Log_event *ev); inline ulonglong until_pos() { DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || @@ -427,7 +427,13 @@ public: return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_pos : group_relay_log_pos); } - + inline char *until_name() + { + DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || + until_condition == UNTIL_RELAY_POS); + return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_name : + group_relay_log_name); + } /** Helper function to do after statement completion. @@ -564,6 +570,15 @@ private: relay_log.info had 4 lines. Now it has 5 lines. */ static const int LINES_IN_RELAY_LOG_INFO_WITH_DELAY= 5; + /* + Hint for when to stop event distribution by sql driver thread. + The flag is set ON by a non-group event when this event is in the middle + of a group (e.g a transaction group) so it's too early + to refresh the current-relay-log vs until-log cached comparison result. + And it is checked and to decide whether it's a right time to do so + when the being processed group has been fully scheduled. + */ + bool until_relay_log_names_defer; /* Holds the state of the data in the relay log. diff --git a/sql/slave.cc b/sql/slave.cc index eb95afd140b..1bf83aa9652 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3921,12 +3921,8 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && (ev->server_id != global_system_variables.server_id || rli->replicate_same_server_id) && - rli->is_until_satisfied((rli->get_flag(Relay_log_info::IN_TRANSACTION) || !ev->log_pos) - ? rli->group_master_log_pos - : ev->log_pos - ev->data_written)) + rli->is_until_satisfied(ev)) { - sql_print_information("Slave SQL thread stopped because it reached its" - " UNTIL position %llu", rli->until_pos()); /* Setting abort_slave flag because we do not want additional message about error in query execution to be printed. @@ -5136,10 +5132,14 @@ pthread_handler_t handle_slave_sql(void *arg) } if ((rli->until_condition == Relay_log_info::UNTIL_MASTER_POS || rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && - rli->is_until_satisfied(rli->group_master_log_pos)) + rli->is_until_satisfied(NULL)) { sql_print_information("Slave SQL thread stopped because it reached its" - " UNTIL position %llu", rli->until_pos()); + " UNTIL position %llu in %s %s file", + rli->until_pos(), rli->until_name(), + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); mysql_mutex_unlock(&rli->data_lock); goto err; } @@ -5205,7 +5205,24 @@ pthread_handler_t handle_slave_sql(void *arg) err: if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); + /* Gtid_list_log_event::do_apply_event has already reported the GTID until */ + if (rli->stop_for_until && rli->until_condition != Relay_log_info::UNTIL_GTID) + { + if (global_system_variables.log_warnings > 2) + sql_print_information("Slave SQL thread UNTIL stop was requested at position " + "%llu in %s %s file", + rli->until_log_pos, rli->until_log_name, + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); + sql_print_information("Slave SQL thread stopped because it reached its" + " UNTIL position %llu in %s %s file", + rli->until_pos(), rli->until_name(), + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); + }; /* Thread stopped. Print the current replication position to the log */ { StringBuffer<100> tmp; From 403dacf6a9a373c25a808a1be9169ab2cd001448 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 26 May 2020 20:04:47 +0300 Subject: [PATCH 08/37] Fixed crash in aria recovery when using bulk insert MDEV-20578 Got error 126 when executing undo undo_key_delete upon Aria crash recovery The crash happens in this scenario: - Table with unique keys and non unique keys - Batch insert (LOAD DATA or INSERT ... SELECT) with REPLACE - Some insert succeeds followed by duplicate key error In the above scenario the table gets corrupted. The bug was that we don't generate any undo entry for the failed insert as the whole insert can be ignored by undo. The code did however not take into account that when bulk insert is used, we would write cached keys to the file on failure and undo would wrongly ignore these. Fixed by moving the writing of the cache keys after we write the aborted-insert event to the log. --- mysql-test/suite/maria/bulk_insert_crash.opt | 2 ++ .../suite/maria/bulk_insert_crash.result | 13 +++++++ mysql-test/suite/maria/bulk_insert_crash.test | 36 +++++++++++++++++++ sql/handler.cc | 13 +++++++ sql/handler.h | 8 +---- storage/maria/ma_recovery.c | 21 +++++++---- storage/maria/ma_write.c | 15 ++++---- 7 files changed, 89 insertions(+), 19 deletions(-) create mode 100644 mysql-test/suite/maria/bulk_insert_crash.opt create mode 100644 mysql-test/suite/maria/bulk_insert_crash.result create mode 100644 mysql-test/suite/maria/bulk_insert_crash.test diff --git a/mysql-test/suite/maria/bulk_insert_crash.opt b/mysql-test/suite/maria/bulk_insert_crash.opt new file mode 100644 index 00000000000..f85a8d9c973 --- /dev/null +++ b/mysql-test/suite/maria/bulk_insert_crash.opt @@ -0,0 +1,2 @@ +--skip-stack-trace --skip-core-file +--default-storage-engine=Aria diff --git a/mysql-test/suite/maria/bulk_insert_crash.result b/mysql-test/suite/maria/bulk_insert_crash.result new file mode 100644 index 00000000000..fc28bf325ba --- /dev/null +++ b/mysql-test/suite/maria/bulk_insert_crash.result @@ -0,0 +1,13 @@ +create table t1 (a int primary key, b int, c int, unique key(b), key(c)) engine=aria transactional=1; +insert into t1 values (1000,1000,1000); +insert into t1 select seq,seq+100, seq+200 from seq_1_to_10; +SET GLOBAL debug_dbug="+d,crash_end_bulk_insert"; +REPLACE into t1 select seq+20,seq+95, seq + 300 from seq_1_to_10; +ERROR HY000: Lost connection to MySQL server during query +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select sum(a),sum(b),sum(c) from t1; +sum(a) sum(b) sum(c) +1055 2055 3055 +drop table t1; diff --git a/mysql-test/suite/maria/bulk_insert_crash.test b/mysql-test/suite/maria/bulk_insert_crash.test new file mode 100644 index 00000000000..d9167c3f0d7 --- /dev/null +++ b/mysql-test/suite/maria/bulk_insert_crash.test @@ -0,0 +1,36 @@ +--source include/not_embedded.inc +--source include/not_valgrind.inc +# Avoid CrashReporter popup on Mac +--source include/not_crashrep.inc +# Binary must be compiled with debug for crash to occur +--source include/have_debug.inc +--source include/have_sequence.inc + +# +# MDEV-20578 Got error 126 when executing undo undo_key_delete upon Aria crash +# recovery +# + +# Write file to make mysql-test-run.pl expect crash and restart +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +create table t1 (a int primary key, b int, c int, unique key(b), key(c)) engine=aria transactional=1; +insert into t1 values (1000,1000,1000); +insert into t1 select seq,seq+100, seq+200 from seq_1_to_10; + +# Insert into t1 with batch insert where we get a rows replaced after +# a few sucessful inserts + +SET GLOBAL debug_dbug="+d,crash_end_bulk_insert"; + +--error 2013 +REPLACE into t1 select seq+20,seq+95, seq + 300 from seq_1_to_10; + +# Wait until restarted +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect + +check table t1; +select sum(a),sum(b),sum(c) from t1; +drop table t1; diff --git a/sql/handler.cc b/sql/handler.cc index d739ce7cd5b..d6e1680143c 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4164,6 +4164,19 @@ int handler::ha_repair(THD* thd, HA_CHECK_OPT* check_opt) } +/** + End bulk insert +*/ + +int handler::ha_end_bulk_insert() +{ + DBUG_ENTER("handler::ha_end_bulk_insert"); + DBUG_EXECUTE_IF("crash_end_bulk_insert", + { extra(HA_EXTRA_FLUSH) ; DBUG_SUICIDE();}); + estimation_rows_to_insert= 0; + DBUG_RETURN(end_bulk_insert()); +} + /** Bulk update row: public interface. diff --git a/sql/handler.h b/sql/handler.h index a0a43daf144..d1c4b79ee48 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -2911,13 +2911,7 @@ public: start_bulk_insert(rows, flags); DBUG_VOID_RETURN; } - int ha_end_bulk_insert() - { - DBUG_ENTER("handler::ha_end_bulk_insert"); - estimation_rows_to_insert= 0; - int ret= end_bulk_insert(); - DBUG_RETURN(ret); - } + int ha_end_bulk_insert(); int ha_bulk_update_row(const uchar *old_data, uchar *new_data, uint *dup_key_found); int ha_delete_all_rows(); diff --git a/storage/maria/ma_recovery.c b/storage/maria/ma_recovery.c index c8031030361..96b678b8695 100644 --- a/storage/maria/ma_recovery.c +++ b/storage/maria/ma_recovery.c @@ -949,6 +949,7 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE) char *old_name, *new_name; int error= 1; MARIA_HA *info= NULL; + my_bool from_table_is_crashed= 0; DBUG_ENTER("exec_REDO_LOGREC_REDO_RENAME_TABLE"); if (skip_DDLs) @@ -1018,15 +1019,15 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE) } if (maria_is_crashed(info)) { - tprint(tracef, ", is crashed, can't rename it"); - ALERT_USER(); - goto end; + tprint(tracef, "is crashed, can't be used for rename ; new-name table "); + from_table_is_crashed= 1; } if (close_one_table(info->s->open_file_name.str, rec->lsn) || maria_close(info)) goto end; info= NULL; - tprint(tracef, ", is ok for renaming; new-name table "); + if (!from_table_is_crashed) + tprint(tracef, "is ok for renaming; new-name table "); } else /* one or two files absent, or header corrupted... */ { @@ -1091,11 +1092,19 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE) goto end; info= NULL; /* abnormal situation */ - tprint(tracef, ", exists but is older than record, can't rename it"); + tprint(tracef, "exists but is older than record, can't rename it"); goto end; } else /* one or two files absent, or header corrupted... */ - tprint(tracef, ", can't be opened, probably does not exist"); + tprint(tracef, "can't be opened, probably does not exist"); + + if (from_table_is_crashed) + { + eprint(tracef, "Aborting rename as old table was crashed"); + ALERT_USER(); + goto end; + } + tprint(tracef, ", renaming '%s'", old_name); if (maria_rename(old_name, new_name)) { diff --git a/storage/maria/ma_write.c b/storage/maria/ma_write.c index 126554fffdb..826ff40baaf 100644 --- a/storage/maria/ma_write.c +++ b/storage/maria/ma_write.c @@ -335,12 +335,6 @@ err: my_errno == HA_ERR_NULL_IN_SPATIAL || my_errno == HA_ERR_OUT_OF_MEM) { - if (info->bulk_insert) - { - uint j; - for (j=0 ; j < share->base.keys ; j++) - maria_flush_bulk_insert(info, j); - } info->errkey= i < share->base.keys ? (int) i : -1; /* We delete keys in the reverse order of insertion. This is the order that @@ -366,6 +360,7 @@ err: { if (_ma_ft_del(info,i,buff,record,filepos)) { + fatal_error= 1; if (local_lock_tree) mysql_rwlock_unlock(&keyinfo->root_lock); break; @@ -380,6 +375,7 @@ err: filepos, info->trn->trid))) { + fatal_error= 1; if (local_lock_tree) mysql_rwlock_unlock(&keyinfo->root_lock); break; @@ -399,6 +395,13 @@ err: fatal_error= 1; } + if (info->bulk_insert) + { + uint j; + for (j=0 ; j < share->base.keys ; j++) + maria_flush_bulk_insert(info, j); + } + if (fatal_error) { maria_print_error(info->s, HA_ERR_CRASHED); From 18d8f06f31cbe3913e1c80c0c5120f23e036bd3e Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sun, 17 May 2020 16:24:40 +0300 Subject: [PATCH 09/37] intrusive::list fixes namespace intrusive: removed split class into two: ilist and sized_ilist which has a size field. ilist no more NULLify pointers to bring a slignly better performance. As a consequence, fil_space_t::is_in_unflushed_spaces and fil_space_t::is_in_rotation_list boolean members are needed now. --- include/{intrusive_list.h => ilist.h} | 101 ++++++++++++++++++++------ storage/innobase/fil/fil0crypt.cc | 3 +- storage/innobase/fil/fil0fil.cc | 52 +++++-------- storage/innobase/include/dyn0buf.h | 8 +- storage/innobase/include/fil0fil.h | 22 +++--- 5 files changed, 114 insertions(+), 72 deletions(-) rename include/{intrusive_list.h => ilist.h} (68%) diff --git a/include/intrusive_list.h b/include/ilist.h similarity index 68% rename from include/intrusive_list.h rename to include/ilist.h index 23534596b93..a94f878f49b 100644 --- a/include/intrusive_list.h +++ b/include/ilist.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2019, MariaDB + Copyright (c) 2019, 2020, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -21,26 +21,28 @@ #include #include -namespace intrusive -{ - // Derive your class from this struct to insert to a linked list. -template struct list_node +template struct ilist_node { - list_node(list_node *next= NULL, list_node *prev= NULL) - : next(next), prev(prev) + ilist_node() + : +#ifndef DBUG_OFF + next(NULL), prev(NULL) +#endif { } - list_node *next; - list_node *prev; + ilist_node(ilist_node *next, ilist_node *prev) : next(next), prev(prev) {} + + ilist_node *next; + ilist_node *prev; }; // Modelled after std::list -template class list +template class ilist { public: - typedef list_node ListNode; + typedef ilist_node ListNode; class Iterator; // All containers in C++ should define these types to implement generic @@ -103,10 +105,10 @@ public: private: ListNode *node_; - friend class list; + friend class ilist; }; - list() : sentinel_(&sentinel_, &sentinel_), size_(0) {} + ilist() : sentinel_(&sentinel_, &sentinel_) {} reference front() { return *begin(); } reference back() { return *--end(); } @@ -127,14 +129,18 @@ public: reverse_iterator rend() { return reverse_iterator(begin()); } const_reverse_iterator rend() const { return reverse_iterator(begin()); } - bool empty() const { return size_ == 0; } - size_type size() const { return size_; } + bool empty() const { return sentinel_.next == &sentinel_; } + + // Not implemented because it's O(N) + // size_type size() const + // { + // return static_cast(std::distance(begin(), end())); + // } void clear() { sentinel_.next= &sentinel_; sentinel_.prev= &sentinel_; - size_= 0; } iterator insert(iterator pos, reference value) @@ -148,7 +154,6 @@ public: static_cast(value).prev= prev; static_cast(value).next= curr; - ++size_; return iterator(&value); } @@ -160,13 +165,12 @@ public: prev->next= next; next->prev= prev; - // This is not required for list functioning. But maybe it'll prevent bugs - // and ease debugging. +#ifndef DBUG_OFF ListNode *curr= pos.node_; curr->prev= NULL; curr->next= NULL; +#endif - --size_; return next; } @@ -177,12 +181,63 @@ public: void pop_front() { erase(begin()); } // STL version is O(n) but this is O(1) because an element can't be inserted - // several times in the same intrusive list. + // several times in the same ilist. void remove(reference value) { erase(iterator(&value)); } private: ListNode sentinel_; - size_type size_; }; -} // namespace intrusive +// Similar to ilist but also has O(1) size() method. +template class sized_ilist : public ilist +{ + typedef ilist BASE; + +public: + // All containers in C++ should define these types to implement generic + // container interface. + using typename BASE::const_iterator; + using typename BASE::const_pointer; + using typename BASE::const_reference; + using typename BASE::const_reverse_iterator; + using typename BASE::difference_type; + using typename BASE::iterator; + using typename BASE::pointer; + using typename BASE::reference; + using typename BASE::reverse_iterator; + using typename BASE::size_type; + using typename BASE::value_type; + + sized_ilist() : size_(0) {} + + size_type size() const { return size_; } + + void clear() + { + BASE::clear(); + size_= 0; + } + + iterator insert(iterator pos, reference value) + { + ++size_; + return BASE::insert(pos, value); + } + + iterator erase(iterator pos) + { + --size_; + return BASE::erase(pos); + } + + void push_back(reference value) { insert(BASE::end(), value); } + void pop_back() { erase(BASE::end()); } + + void push_front(reference value) { insert(BASE::begin(), value); } + void pop_front() { erase(BASE::begin()); } + + void remove(reference value) { erase(iterator(&value)); } + +private: + size_t size_; +}; diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index a28c9f63797..f06aa7606ee 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2261,7 +2261,7 @@ static void fil_crypt_rotation_list_fill() space != NULL; space = UT_LIST_GET_NEXT(space_list, space)) { if (space->purpose != FIL_TYPE_TABLESPACE - || space->is_in_rotation_list() + || space->is_in_rotation_list || space->is_stopping() || UT_LIST_GET_LEN(space->chain) == 0) { continue; @@ -2306,6 +2306,7 @@ static void fil_crypt_rotation_list_fill() } fil_system->rotation_list.push_back(*space); + space->is_in_rotation_list = true; } } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index d4b0d3f4377..bb7fc0d015f 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2019, MariaDB Corporation. +Copyright (c) 2014, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -832,7 +832,7 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false) /* No need to flush. User has explicitly disabled buffering. */ - ut_ad(!space->is_in_unflushed_spaces()); + ut_ad(!space->is_in_unflushed_spaces); ut_ad(fil_space_is_flushed(space)); ut_ad(space->n_pending_flushes == 0); @@ -895,10 +895,11 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false) skip_flush: #endif /* _WIN32 */ if (!node->needs_flush) { - if (space->is_in_unflushed_spaces() + if (space->is_in_unflushed_spaces && fil_space_is_flushed(space)) { fil_system->unflushed_spaces.remove(*space); + space->is_in_unflushed_spaces = false; } } @@ -1193,13 +1194,14 @@ fil_node_close_to_free( if (fil_buffering_disabled(space)) { - ut_ad(!space->is_in_unflushed_spaces()); + ut_ad(!space->is_in_unflushed_spaces); ut_ad(fil_space_is_flushed(space)); - } else if (space->is_in_unflushed_spaces() + } else if (space->is_in_unflushed_spaces && fil_space_is_flushed(space)) { fil_system->unflushed_spaces.remove(*space); + space->is_in_unflushed_spaces = false; } fil_node_close_file(node); @@ -1226,16 +1228,18 @@ fil_space_detach( HASH_DELETE(fil_space_t, name_hash, fil_system->name_hash, ut_fold_string(space->name), space); - if (space->is_in_unflushed_spaces()) { + if (space->is_in_unflushed_spaces) { ut_ad(!fil_buffering_disabled(space)); fil_system->unflushed_spaces.remove(*space); + space->is_in_unflushed_spaces = false; } - if (space->is_in_rotation_list()) { + if (space->is_in_rotation_list) { fil_system->rotation_list.remove(*space); + space->is_in_rotation_list = false; } UT_LIST_REMOVE(fil_system->space_list, space); @@ -1462,6 +1466,7 @@ fil_space_create( /* Key rotation is not enabled, need to inform background encryption threads. */ fil_system->rotation_list.push_back(*space); + space->is_in_rotation_list = true; mutex_exit(&fil_system->mutex); mutex_enter(&fil_crypt_threads_mutex); os_event_set(fil_crypt_threads_event); @@ -4751,16 +4756,17 @@ fil_node_complete_io(fil_node_t* node, const IORequest& type) /* We don't need to keep track of unflushed changes as user has explicitly disabled buffering. */ - ut_ad(!node->space->is_in_unflushed_spaces()); + ut_ad(!node->space->is_in_unflushed_spaces); ut_ad(node->needs_flush == false); } else { node->needs_flush = true; - if (!node->space->is_in_unflushed_spaces()) { + if (!node->space->is_in_unflushed_spaces) { fil_system->unflushed_spaces.push_front( *node->space); + node->space->is_in_unflushed_spaces = true; } } } @@ -5247,7 +5253,7 @@ fil_flush_file_spaces( n_space_ids = 0; - for (intrusive::list::iterator it + for (sized_ilist::iterator it = fil_system->unflushed_spaces.begin(), end = fil_system->unflushed_spaces.end(); it != end; ++it) { @@ -5974,9 +5980,10 @@ fil_space_remove_from_keyrotation(fil_space_t* space) ut_ad(mutex_own(&fil_system->mutex)); ut_ad(space); - if (space->n_pending_ops == 0 && space->is_in_rotation_list()) { + if (space->n_pending_ops == 0 && space->is_in_rotation_list) { ut_a(!fil_system->rotation_list.empty()); fil_system->rotation_list.remove(*space); + space->is_in_rotation_list = false; } } @@ -6005,7 +6012,7 @@ fil_space_t *fil_system_t::keyrotate_next(fil_space_t *prev_space, don't remove the last processed tablespace from the rotation list. */ const bool remove= (!recheck || prev_space->crypt_data) && !key_version == !srv_encrypt_tables; - intrusive::list::iterator it= + sized_ilist::iterator it= prev_space == NULL ? fil_system->rotation_list.end() : prev_space; if (it == fil_system->rotation_list.end()) @@ -6117,24 +6124,3 @@ fil_space_set_punch_hole( { node->space->punch_hole = val; } - -/** Checks that this tablespace in a list of unflushed tablespaces. -@return true if in a list */ -bool fil_space_t::is_in_unflushed_spaces() const -{ - ut_ad(mutex_own(&fil_system->mutex)); - - return static_cast *>( - this) - ->next; -} - -/** Checks that this tablespace needs key rotation. -@return true if in a rotation list */ -bool fil_space_t::is_in_rotation_list() const -{ - ut_ad(mutex_own(&fil_system->mutex)); - - return static_cast *>(this) - ->next; -} diff --git a/storage/innobase/include/dyn0buf.h b/storage/innobase/include/dyn0buf.h index 311f6518943..b3302409e5f 100644 --- a/storage/innobase/include/dyn0buf.h +++ b/storage/innobase/include/dyn0buf.h @@ -29,7 +29,7 @@ Created 2013-03-16 Sunny Bains #include "mem0mem.h" #include "dyn0types.h" -#include "intrusive_list.h" +#include "ilist.h" /** Class that manages dynamic buffers. It uses a UT_LIST of @@ -43,7 +43,7 @@ template class dyn_buf_t { public: - class block_t : public intrusive::list_node<> { + class block_t : public ilist_node<> { public: block_t() @@ -153,7 +153,7 @@ public: /** SIZE - sizeof(m_node) + sizeof(m_used) */ enum { MAX_DATA_SIZE = SIZE - - sizeof(intrusive::list_node<>) + - sizeof(ilist_node<>) + sizeof(ib_uint32_t) }; @@ -167,7 +167,7 @@ public: friend class dyn_buf_t; }; - typedef intrusive::list list_t; + typedef sized_ilist list_t; enum { MAX_DATA_SIZE = block_t::MAX_DATA_SIZE}; diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9c722944665..62d87ce2c06 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -33,7 +33,7 @@ Created 10/25/1995 Heikki Tuuri #include "dict0types.h" #include "page0size.h" #include "ibuf0types.h" -#include "intrusive_list.h" +#include "ilist.h" #include @@ -82,8 +82,8 @@ fil_type_is_data( struct fil_node_t; /** Tablespace or log data space */ -struct fil_space_t : intrusive::list_node, - intrusive::list_node +struct fil_space_t : ilist_node, + ilist_node { ulint id; /*!< space id */ hash_node_t hash; /*!< hash chain node */ @@ -165,18 +165,18 @@ struct fil_space_t : intrusive::list_node, UT_LIST_NODE_T(fil_space_t) named_spaces; /*!< list of spaces for which MLOG_FILE_NAME records have been issued */ - /** Checks that this tablespace in a list of unflushed tablespaces. - @return true if in a list */ - bool is_in_unflushed_spaces() const; UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - /** Checks that this tablespace needs key rotation. - @return true if in a rotation list */ - bool is_in_rotation_list() const; /** MariaDB encryption data */ fil_space_crypt_t* crypt_data; + /** Checks that this tablespace in a list of unflushed tablespaces. */ + bool is_in_unflushed_spaces; + + /** Checks that this tablespace needs key rotation. */ + bool is_in_rotation_list; + /** True if the device this filespace is on supports atomic writes */ bool atomic_write_supported; @@ -507,7 +507,7 @@ struct fil_system_t { not put to this list: they are opened after the startup, and kept open until shutdown */ - intrusive::list unflushed_spaces; + sized_ilist unflushed_spaces; /*!< list of those tablespaces whose files contain unflushed writes; those spaces have @@ -530,7 +530,7 @@ struct fil_system_t { record has been written since the latest redo log checkpoint. Protected only by log_sys->mutex. */ - intrusive::list rotation_list; + ilist rotation_list; /*!< list of all file spaces needing key rotation.*/ From 67496281eaa11098ad4482c2bb5aa5976b4f5cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 27 May 2020 09:51:46 +0300 Subject: [PATCH 10/37] Fix the RelWithDebInfo build The build was broken in the parent commit 18d8f06f31cbe3913e1c80c0c5120f23e036bd3e. --- include/ilist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ilist.h b/include/ilist.h index a94f878f49b..ae87b79d54e 100644 --- a/include/ilist.h +++ b/include/ilist.h @@ -25,8 +25,8 @@ template struct ilist_node { ilist_node() - : #ifndef DBUG_OFF + : next(NULL), prev(NULL) #endif { From bf1aa7569e7a29fae2b1cdf36b840e98130fda4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 27 May 2020 09:53:36 +0300 Subject: [PATCH 11/37] Add an end-of-test marker to ease merges --- mysql-test/r/partition_alter.result | 1 + mysql-test/t/partition_alter.test | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mysql-test/r/partition_alter.result b/mysql-test/r/partition_alter.result index 8de3d4b45ad..699377751d1 100644 --- a/mysql-test/r/partition_alter.result +++ b/mysql-test/r/partition_alter.result @@ -165,3 +165,4 @@ select * from t1 partition (p0); k x 1 2000-01-01 00:00:00.000000 drop table t1; +# End of 10.2 tests diff --git a/mysql-test/t/partition_alter.test b/mysql-test/t/partition_alter.test index 3c783ebe375..b52e92445a8 100644 --- a/mysql-test/t/partition_alter.test +++ b/mysql-test/t/partition_alter.test @@ -151,3 +151,5 @@ insert into t1 (k, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); alter table t1 drop key u, algorithm=inplace; select * from t1 partition (p0); drop table t1; + +--echo # End of 10.2 tests From 1b3adaab25c4f0e4288b261f973906bbf6b394c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 27 May 2020 10:58:28 +0300 Subject: [PATCH 12/37] Fix the build with GCC 4.1.2 The build was broken on CentOS 5 and CentOS 6 as a result of commit 18d8f06f31cbe3913e1c80c0c5120f23e036bd3e introducing some constructs that old GCC versions could not cope with. --- include/ilist.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/include/ilist.h b/include/ilist.h index ae87b79d54e..746061c397d 100644 --- a/include/ilist.h +++ b/include/ilist.h @@ -210,7 +210,7 @@ public: sized_ilist() : size_(0) {} - size_type size() const { return size_; } + std::size_t size() const { return size_; } void clear() { @@ -218,25 +218,30 @@ public: size_= 0; } - iterator insert(iterator pos, reference value) + typename ilist::iterator + insert(typename ilist::iterator pos, + typename ilist::reference value) { ++size_; return BASE::insert(pos, value); } - iterator erase(iterator pos) + typename ilist::iterator erase(typename ilist::iterator pos) { --size_; return BASE::erase(pos); } - void push_back(reference value) { insert(BASE::end(), value); } + void push_back(typename ilist::reference value) + { insert(BASE::end(), value); } void pop_back() { erase(BASE::end()); } - void push_front(reference value) { insert(BASE::begin(), value); } + void push_front(typename ilist::reference value) + { insert(BASE::begin(), value); } void pop_front() { erase(BASE::begin()); } - void remove(reference value) { erase(iterator(&value)); } + void remove(typename ilist::reference value) + { erase(iterator(&value)); } private: size_t size_; From 5139cfabb3bfc62be38e651ff176580e0aba09b8 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 27 May 2020 12:59:27 +0300 Subject: [PATCH 13/37] fix compilation --- include/ilist.h | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/include/ilist.h b/include/ilist.h index 746061c397d..5d0a425a9cc 100644 --- a/include/ilist.h +++ b/include/ilist.h @@ -196,21 +196,21 @@ template class sized_ilist : public ilist public: // All containers in C++ should define these types to implement generic // container interface. - using typename BASE::const_iterator; - using typename BASE::const_pointer; - using typename BASE::const_reference; - using typename BASE::const_reverse_iterator; - using typename BASE::difference_type; - using typename BASE::iterator; - using typename BASE::pointer; - using typename BASE::reference; - using typename BASE::reverse_iterator; - using typename BASE::size_type; - using typename BASE::value_type; + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type &reference; + typedef const value_type &const_reference; + typedef T *pointer; + typedef const T *const_pointer; + typedef typename BASE::Iterator iterator; + typedef const typename BASE::Iterator const_iterator; + typedef std::reverse_iterator reverse_iterator; + typedef std::reverse_iterator const_reverse_iterator; sized_ilist() : size_(0) {} - std::size_t size() const { return size_; } + size_type size() const { return size_; } void clear() { @@ -218,31 +218,26 @@ public: size_= 0; } - typename ilist::iterator - insert(typename ilist::iterator pos, - typename ilist::reference value) + iterator insert(iterator pos, reference value) { ++size_; return BASE::insert(pos, value); } - typename ilist::iterator erase(typename ilist::iterator pos) + iterator erase(iterator pos) { --size_; return BASE::erase(pos); } - void push_back(typename ilist::reference value) - { insert(BASE::end(), value); } + void push_back(reference value) { insert(BASE::end(), value); } void pop_back() { erase(BASE::end()); } - void push_front(typename ilist::reference value) - { insert(BASE::begin(), value); } + void push_front(reference value) { insert(BASE::begin(), value); } void pop_front() { erase(BASE::begin()); } - void remove(typename ilist::reference value) - { erase(iterator(&value)); } + void remove(reference value) { erase(iterator(&value)); } private: - size_t size_; + size_type size_; }; From a50e6c9eb126cbb1d46735dade6c9006f1a5a764 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 13 May 2020 13:10:35 +0200 Subject: [PATCH 14/37] MDEV-17153 server crash on repair table ... use_frm data_file_length == 0 in mi_repair() is normal for REPAIR ... USE_FRM. But in-file links (for blocks and deleted chain) must be compared with the real file length to avoid spurious "link points outside datafile" warnings and arbitrary block skipping. --- mysql-test/r/repair.result | 21 +++++++++++++++++++++ mysql-test/t/repair.test | 21 +++++++++++++++++++++ storage/myisam/mi_check.c | 6 ++++++ 3 files changed, 48 insertions(+) diff --git a/mysql-test/r/repair.result b/mysql-test/r/repair.result index 171ef836bda..749b8a74b4f 100644 --- a/mysql-test/r/repair.result +++ b/mysql-test/r/repair.result @@ -233,3 +233,24 @@ i UNLOCK TABLES; DROP TABLE t1; # End of 10.0 tests +create table t1 (a int, b varchar(200)); +insert t1 select seq, repeat(200, seq) from seq_1_to_30; +delete from t1 where a % 13 = 0; +repair table t1 use_frm; +Table Op Msg_type Msg_text +test.t1 repair warning Number of rows changed from 0 to 28 +test.t1 repair status OK +delete from t1 where a % 11 = 0; +repair table t1 extended use_frm; +Table Op Msg_type Msg_text +test.t1 repair warning Number of rows changed from 0 to 26 +test.t1 repair status OK +delete from t1 where a % 7 = 0; +set myisam_repair_threads = 2; +repair table t1 use_frm; +Table Op Msg_type Msg_text +test.t1 repair warning Number of rows changed from 0 to 22 +test.t1 repair status OK +set myisam_repair_threads = default; +drop table t1; +# End of 10.2 tests diff --git a/mysql-test/t/repair.test b/mysql-test/t/repair.test index 75978ac482b..435eb99683a 100644 --- a/mysql-test/t/repair.test +++ b/mysql-test/t/repair.test @@ -1,6 +1,7 @@ # # Test of repair table # +--source include/have_sequence.inc --source include/default_charset.inc call mtr.add_suppression("character set is multi-byte"); @@ -246,3 +247,23 @@ UNLOCK TABLES; DROP TABLE t1; --echo # End of 10.0 tests + +# +# MDEV-17153 server crash on repair table ... use_frm +# +# Note, this test case doesn't crash, but shows spurios warnings +# unless the bug is fixed +# +create table t1 (a int, b varchar(200)); +insert t1 select seq, repeat(200, seq) from seq_1_to_30; +delete from t1 where a % 13 = 0; +repair table t1 use_frm; +delete from t1 where a % 11 = 0; +repair table t1 extended use_frm; +delete from t1 where a % 7 = 0; +set myisam_repair_threads = 2; +repair table t1 use_frm; +set myisam_repair_threads = default; +drop table t1; + +--echo # End of 10.2 tests diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index b18ffb99a11..d9b9bb5af4a 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -1586,6 +1586,8 @@ int mi_repair(HA_CHECK *param, register MI_INFO *info, sort_param.filepos=new_header_length; param->read_cache.end_of_file=sort_info.filelength= mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; sort_info.dupp=0; sort_param.fix_datafile= (my_bool) (! rep_quick); sort_param.master=1; @@ -2290,6 +2292,8 @@ int mi_repair_by_sort(HA_CHECK *param, register MI_INFO *info, sort_info.buff=0; param->read_cache.end_of_file=sort_info.filelength= mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; sort_param.wordlist=NULL; init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0, @@ -2757,6 +2761,8 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info, sort_info.buff=0; param->read_cache.end_of_file=sort_info.filelength= mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; if (share->data_file_type == DYNAMIC_RECORD) rec_length=MY_MAX(share->base.min_pack_length+1,share->base.min_block_length); From 39c141b4ae9e6381f9efd4c566e61b91371a3d63 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 13 May 2020 19:48:23 +0200 Subject: [PATCH 15/37] don't include .git files in source packages the existing ".git/" rule only filters out .git directories, but as we have submodules now, we have to filter out .git files too. --- cmake/cpack_source_ignore_files.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/cpack_source_ignore_files.cmake b/cmake/cpack_source_ignore_files.cmake index 5b103501a99..eb1ec191d17 100644 --- a/cmake/cpack_source_ignore_files.cmake +++ b/cmake/cpack_source_ignore_files.cmake @@ -15,6 +15,7 @@ SET(CPACK_SOURCE_IGNORE_FILES \\\\.git/ +\\\\.git$ \\\\.gitignore$ \\\\.gitattributes$ CMakeCache\\\\.txt$ From b01c8a6cc8b361c6c8ecd887f1958c1f0b2e9811 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 14 May 2020 16:18:01 +0200 Subject: [PATCH 16/37] MDEV-22558 wrong error for invalid utf8 table comment --- mysql-test/r/comment_table.result | 5 ++++- mysql-test/t/comment_table.test | 11 ++++++++--- sql/sql_table.cc | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/comment_table.result b/mysql-test/r/comment_table.result index 99ecb04a362..d0af71b4540 100644 --- a/mysql-test/r/comment_table.result +++ b/mysql-test/r/comment_table.result @@ -1,4 +1,3 @@ -DROP TABLE IF EXISTS t1; create table t1 (c1 VARCHAR(10) NOT NULL COMMENT 'c1 comment', c2 INTEGER,c3 INTEGER COMMENT '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789', c4 INTEGER, c5 INTEGER, c6 INTEGER, c7 INTEGER, INDEX i1 (c1) COMMENT 'i1 comment',INDEX i2(c2) ) COMMENT='abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcd'; SELECT table_comment,char_length(table_comment) FROM information_schema.tables WHERE table_name='t1'; @@ -125,3 +124,7 @@ SELECT table_comment,char_length(table_comment) FROM information_schema.tables W table_comment char_length(table_comment) SELECT column_comment,char_length(column_comment) FROM information_schema.columns WHERE table_name='t1'; column_comment char_length(column_comment) +set names utf8; +create table t1 (x int comment 'a’'); +ERROR HY000: Invalid utf8 character string: 'a' +set names latin1; diff --git a/mysql-test/t/comment_table.test b/mysql-test/t/comment_table.test index 2a85fbac571..db8bdf54ba6 100644 --- a/mysql-test/t/comment_table.test +++ b/mysql-test/t/comment_table.test @@ -1,6 +1,3 @@ ---disable_warnings -DROP TABLE IF EXISTS t1; ---enable_warnings #1024 bytes create table t1 (c1 VARCHAR(10) NOT NULL COMMENT 'c1 comment', c2 INTEGER,c3 INTEGER COMMENT '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789', c4 INTEGER, c5 INTEGER, c6 INTEGER, c7 INTEGER, INDEX i1 (c1) COMMENT 'i1 comment',INDEX i2(c2) ) COMMENT='abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcd'; @@ -53,3 +50,11 @@ create table t1 (c1 VARCHAR(10) NOT NULL COMMENT 'c1 comment', c2 INTEGER,c3 INT ) COMMENT='abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcde'; SELECT table_comment,char_length(table_comment) FROM information_schema.tables WHERE table_name='t1'; SELECT column_comment,char_length(column_comment) FROM information_schema.columns WHERE table_name='t1'; + +# +# MDEV-22558 wrong error for invalid utf8 table comment +# +set names utf8; +--error ER_INVALID_CHARACTER_STRING +create table t1 (x int comment 'a’'); +set names latin1; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 8d81911bf29..87d84b1abc6 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4294,6 +4294,25 @@ bool validate_comment_length(THD *thd, LEX_STRING *comment, size_t max_len, Well_formed_prefix(system_charset_info, *comment, max_len).length(); if (tmp_len < comment->length) { +#if MARIADB_VERSION_ID < 100500 + if (comment->length <= max_len) + { + if (thd->is_strict_mode()) + { + my_error(ER_INVALID_CHARACTER_STRING, MYF(0), + system_charset_info->csname, comment->str); + DBUG_RETURN(true); + } + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_INVALID_CHARACTER_STRING, + ER_THD(thd, ER_INVALID_CHARACTER_STRING), + system_charset_info->csname, comment->str); + comment->length= tmp_len; + DBUG_RETURN(false); + } +#else +#error do it in TEXT_STRING_sys +#endif if (thd->is_strict_mode()) { my_error(err_code, MYF(0), name, static_cast(max_len)); From 1e951155bddd42b84e1c4ed949a85d62829e696f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 May 2020 17:02:47 +0200 Subject: [PATCH 17/37] bugfix: use THD::main_mem_root for kill error message cannot use the current THD::mem_root, because it can be temporarily reassigned to something with a very different life time (e.g. to TABLE::mem_root or range optimizer mem_root). --- sql/sql_class.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/sql_class.h b/sql/sql_class.h index 80aa5710b62..838998af94f 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3729,7 +3729,8 @@ public: The worst things that can happen is that we get a suboptimal error message. */ - if ((killed_err= (err_info*) alloc(sizeof(*killed_err)))) + killed_err= (err_info*) alloc_root(&main_mem_root, sizeof(*killed_err)); + if (killed_err) { killed_err->no= killed_errno_arg; ::strmake((char*) killed_err->msg, killed_err_msg_arg, From ad772478660a67c1c93c22725e25b168ec512f4d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 May 2020 17:04:22 +0200 Subject: [PATCH 18/37] MDEV-21958 Query having many NOT-IN clauses running forever and causing available free memory to use completely let thd->killed to abort range optimizer --- sql/opt_range.h | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/opt_range.h b/sql/opt_range.h index dff50252e1a..1e6aca540d8 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -656,6 +656,7 @@ public: bool statement_should_be_aborted() const { return + thd->killed || thd->is_fatal_error || thd->is_error() || alloced_sel_args > SEL_ARG::MAX_SEL_ARGS; From 6af37ba881fee7e6f651d5e0730c9374337ad1b4 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 21 May 2020 01:31:17 +0200 Subject: [PATCH 19/37] fix rocksdb zstd detection WITH_ROCKSDB_ZSTD must use the same capitalization as in the foreach loop --- storage/rocksdb/build_rocksdb.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index ca61842fbb8..3f3dca7e990 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -67,7 +67,7 @@ if(SNAPPY_FOUND AND (NOT WITH_ROCKSDB_snappy STREQUAL "OFF")) endif() include(CheckFunctionExists) -if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_zstd STREQUAL "OFF")) +if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_ZSTD STREQUAL "OFF")) SET(CMAKE_REQUIRED_LIBRARIES zstd) CHECK_FUNCTION_EXISTS(ZDICT_trainFromBuffer ZSTD_VALID) UNSET(CMAKE_REQUIRED_LIBRARIES) From cceb965a7918c1e3a29965fe0b440cf051764358 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 May 2020 12:05:53 +0200 Subject: [PATCH 20/37] Revert "MDEV-12445 : Rocksdb does not shutdown worker threads and aborts in memleak check on server shutdown" This reverts commit 6f1f9114971c3c158e9ac97313c92a24f6554415. because it doesn't do anything now (the server doesn't check my_disable_leak_check) and it never did anything before (because without `extern` it simply created a local instance of my_disable_leak_check, did not affect server's my_disable_leak_check). --- sql/mysqld.cc | 1 - storage/rocksdb/ha_rocksdb.cc | 8 -------- 2 files changed, 9 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5878f4a2286..e40d5c38233 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -365,7 +365,6 @@ static bool volatile select_thread_in_use, signal_thread_in_use; static volatile bool ready_to_exit; static my_bool opt_debugging= 0, opt_external_locking= 0, opt_console= 0; static my_bool opt_short_log_format= 0, opt_silent_startup= 0; -bool my_disable_leak_check= false; uint kill_cached_threads; static uint wake_thread; diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index bf9183d7dea..5cbe4c162a8 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -114,7 +114,6 @@ int thd_binlog_format(const MYSQL_THD thd); bool thd_binlog_filter_ok(const MYSQL_THD thd); } -MYSQL_PLUGIN_IMPORT bool my_disable_leak_check; extern my_bool opt_core_file; // Needed in rocksdb_init_func @@ -5688,13 +5687,6 @@ static int rocksdb_init_func(void *const p) { } #endif - /** - Rocksdb does not always shutdown its threads, when - plugin is shut down. Disable server's leak check - at exit to avoid crash. - */ - my_disable_leak_check = true; - err = my_error_register(rdb_get_error_messages, HA_ERR_ROCKSDB_FIRST, HA_ERR_ROCKSDB_LAST); if (err != 0) { From 8cf589218f1b6ac5f8775068873357bd0dc8acbb Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 May 2020 19:50:22 +0200 Subject: [PATCH 21/37] optimize performance of the build in a fresh clone don't fetch the complete history of all submodules, it's rarely needed --- cmake/submodules.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/submodules.cmake b/cmake/submodules.cmake index 34d1f37c956..6e039bfeae3 100644 --- a/cmake/submodules.cmake +++ b/cmake/submodules.cmake @@ -19,11 +19,11 @@ IF(GIT_EXECUTABLE AND EXISTS "${CMAKE_SOURCE_DIR}/.git") SET(update_result 0) ELSEIF (cmake_update_submodules MATCHES force) MESSAGE(STATUS "Updating submodules (forced)") - EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --force + EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --force --depth=1 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE update_result) ELSEIF (cmake_update_submodules MATCHES yes) - EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init + EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --depth=1 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE update_result) ELSE() From 04726f292088cf00c1ac9ed7543535e7a091810a Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 23 May 2020 18:35:42 +0200 Subject: [PATCH 22/37] get rid of cmake warning --- cmake/build_configurations/mysql_release.cmake | 2 +- .../tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index b19a9437495..1609c45a632 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -136,7 +136,7 @@ IF(UNIX) RedHat/Fedora/Oracle Linux: yum install libaio-devel SuSE: zypper install libaio-devel - If you really do not want it, pass -DIGNORE_AIO_CHECK to cmake. + If you really do not want it, pass -DIGNORE_AIO_CHECK=ON to cmake. ") ENDIF() diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake index 2f04a33558a..88210868eda 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake @@ -134,4 +134,4 @@ static __thread int tlsvar = 0; int main(void) { return tlsvar; }" HAVE_GNU_TLS) ## set TOKUDB_REVISION -set(CMAKE_TOKUDB_REVISION 0 CACHE INTEGER "Revision of tokudb.") +set(CMAKE_TOKUDB_REVISION 0 CACHE STRING "Revision of tokudb.") From e64dc07125c8143e27565947e71db76d597dfaf8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 24 May 2020 21:10:41 +0200 Subject: [PATCH 23/37] assert(a && b); -> assert(a); assert(b); --- mysys/my_bitmap.c | 56 +++++++++++++++++++++++++++------------------ sql/ha_partition.cc | 17 ++++++++------ 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/mysys/my_bitmap.c b/mysys/my_bitmap.c index c79c5a2d324..f857101af02 100644 --- a/mysys/my_bitmap.c +++ b/mysys/my_bitmap.c @@ -255,7 +255,8 @@ my_bool bitmap_fast_test_and_set(MY_BITMAP *map, uint bitmap_bit) my_bool bitmap_test_and_set(MY_BITMAP *map, uint bitmap_bit) { my_bool res; - DBUG_ASSERT(map->bitmap && bitmap_bit < map->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(bitmap_bit < map->n_bits); bitmap_lock(map); res= bitmap_fast_test_and_set(map, bitmap_bit); bitmap_unlock(map); @@ -288,7 +289,8 @@ my_bool bitmap_fast_test_and_clear(MY_BITMAP *map, uint bitmap_bit) my_bool bitmap_test_and_clear(MY_BITMAP *map, uint bitmap_bit) { my_bool res; - DBUG_ASSERT(map->bitmap && bitmap_bit < map->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(bitmap_bit < map->n_bits); bitmap_lock(map); res= bitmap_fast_test_and_clear(map, bitmap_bit); bitmap_unlock(map); @@ -317,8 +319,8 @@ void bitmap_set_prefix(MY_BITMAP *map, uint prefix_size) uint prefix_bytes, prefix_bits, d; uchar *m= (uchar *)map->bitmap; - DBUG_ASSERT(map->bitmap && - (prefix_size <= map->n_bits || prefix_size == (uint) ~0)); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(prefix_size <= map->n_bits || prefix_size == (uint) ~0); set_if_smaller(prefix_size, map->n_bits); if ((prefix_bytes= prefix_size / 8)) memset(m, 0xff, prefix_bytes); @@ -340,7 +342,8 @@ my_bool bitmap_is_prefix(const MY_BITMAP *map, uint prefix_size) uchar *m= (uchar*) map->bitmap; uchar *end_prefix= m+(prefix_size-1)/8; uchar *end; - DBUG_ASSERT(m && prefix_size <= map->n_bits); + DBUG_ASSERT(m); + DBUG_ASSERT(prefix_size <= map->n_bits); /* Empty prefix is always true */ if (!prefix_size) @@ -393,8 +396,8 @@ my_bool bitmap_is_subset(const MY_BITMAP *map1, const MY_BITMAP *map2) { my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end; - DBUG_ASSERT(map1->bitmap && map2->bitmap && - map1->n_bits==map2->n_bits); + DBUG_ASSERT(map1->bitmap && map2->bitmap); + DBUG_ASSERT(map1->n_bits==map2->n_bits); end= map1->last_word_ptr; while (m1 < end) @@ -412,8 +415,9 @@ my_bool bitmap_is_overlapping(const MY_BITMAP *map1, const MY_BITMAP *map2) { my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end; - DBUG_ASSERT(map1->bitmap && map2->bitmap && - map1->n_bits==map2->n_bits); + DBUG_ASSERT(map1->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map1->n_bits==map2->n_bits); end= map1->last_word_ptr; while (m1 < end) @@ -431,7 +435,8 @@ void bitmap_intersect(MY_BITMAP *map, const MY_BITMAP *map2) my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end; uint len= no_words_in_map(map), len2 = no_words_in_map(map2); - DBUG_ASSERT(map->bitmap && map2->bitmap); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); end= to+MY_MIN(len,len2); while (to < end) @@ -476,7 +481,8 @@ my_bool bitmap_exists_intersection(const MY_BITMAP **bitmap_array, uint i, j, start_idx, end_idx; my_bitmap_map cur_res; - DBUG_ASSERT(bitmap_count && end_bit >= start_bit); + DBUG_ASSERT(bitmap_count); + DBUG_ASSERT(end_bit >= start_bit); for (j= 0; j < bitmap_count; j++) DBUG_ASSERT(end_bit < bitmap_array[j]->n_bits); @@ -504,8 +510,9 @@ my_bool bitmap_union_is_set_all(const MY_BITMAP *map1, const MY_BITMAP *map2) { my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end; - DBUG_ASSERT(map1->bitmap && map2->bitmap && - map1->n_bits==map2->n_bits); + DBUG_ASSERT(map1->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map1->n_bits==map2->n_bits); end= map1->last_word_ptr; while ( m1 < end) if ((*m1++ | *m2++) != 0xFFFFFFFF) @@ -550,8 +557,9 @@ void bitmap_set_above(MY_BITMAP *map, uint from_byte, uint use_bit) void bitmap_subtract(MY_BITMAP *map, const MY_BITMAP *map2) { my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end; - DBUG_ASSERT(map->bitmap && map2->bitmap && - map->n_bits==map2->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map->n_bits==map2->n_bits); end= map->last_word_ptr; @@ -564,8 +572,9 @@ void bitmap_union(MY_BITMAP *map, const MY_BITMAP *map2) { my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end; - DBUG_ASSERT(map->bitmap && map2->bitmap && - map->n_bits==map2->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map->n_bits == map2->n_bits); end= map->last_word_ptr; while (to <= end) @@ -576,8 +585,9 @@ void bitmap_union(MY_BITMAP *map, const MY_BITMAP *map2) void bitmap_xor(MY_BITMAP *map, const MY_BITMAP *map2) { my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end= map->last_word_ptr; - DBUG_ASSERT(map->bitmap && map2->bitmap && - map->n_bits==map2->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map->n_bits == map2->n_bits); while (to <= end) *to++ ^= *from++; } @@ -614,8 +624,9 @@ void bitmap_copy(MY_BITMAP *map, const MY_BITMAP *map2) { my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end; - DBUG_ASSERT(map->bitmap && map2->bitmap && - map->n_bits==map2->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map->n_bits == map2->n_bits); end= map->last_word_ptr; while (to <= end) @@ -744,7 +755,8 @@ uint bitmap_lock_set_next(MY_BITMAP *map) void bitmap_lock_clear_bit(MY_BITMAP *map, uint bitmap_bit) { bitmap_lock(map); - DBUG_ASSERT(map->bitmap && bitmap_bit < map->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(bitmap_bit < map->n_bits); bitmap_clear_bit(map, bitmap_bit); bitmap_unlock(map); } diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 15e3a765075..453a1c9ff89 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -2183,7 +2183,8 @@ void ha_partition::update_create_info(HA_CREATE_INFO *create_info) sub_elem= subpart_it++; DBUG_ASSERT(sub_elem); part= i * num_subparts + j; - DBUG_ASSERT(part < m_file_tot_parts && m_file[part]); + DBUG_ASSERT(part < m_file_tot_parts); + DBUG_ASSERT(m_file[part]); dummy_info.data_file_name= dummy_info.index_file_name = NULL; m_file[part]->update_create_info(&dummy_info); sub_elem->data_file_name = (char*) dummy_info.data_file_name; @@ -3770,7 +3771,8 @@ int ha_partition::external_lock(THD *thd, int lock_type) MY_BITMAP *used_partitions; DBUG_ENTER("ha_partition::external_lock"); - DBUG_ASSERT(!auto_increment_lock && !auto_increment_safe_stmt_log_lock); + DBUG_ASSERT(!auto_increment_lock); + DBUG_ASSERT(!auto_increment_safe_stmt_log_lock); if (lock_type == F_UNLCK) used_partitions= &m_locked_partitions; @@ -4034,8 +4036,8 @@ void ha_partition::unlock_row() bool ha_partition::was_semi_consistent_read() { DBUG_ENTER("ha_partition::was_semi_consistent_read"); - DBUG_ASSERT(m_last_part < m_tot_parts && - bitmap_is_set(&(m_part_info->read_partitions), m_last_part)); + DBUG_ASSERT(m_last_part < m_tot_parts); + DBUG_ASSERT(bitmap_is_set(&(m_part_info->read_partitions), m_last_part)); DBUG_RETURN(m_file[m_last_part]->was_semi_consistent_read()); } @@ -5910,8 +5912,8 @@ int ha_partition::partition_scan_set_up(uchar * buf, bool idx_read_flag) DBUG_ASSERT(m_part_spec.start_part < m_tot_parts); m_ordered_scan_ongoing= m_ordered; } - DBUG_ASSERT(m_part_spec.start_part < m_tot_parts && - m_part_spec.end_part < m_tot_parts); + DBUG_ASSERT(m_part_spec.start_part < m_tot_parts); + DBUG_ASSERT(m_part_spec.end_part < m_tot_parts); DBUG_RETURN(0); } @@ -8653,7 +8655,8 @@ void ha_partition::get_auto_increment(ulonglong offset, ulonglong increment, DBUG_PRINT("info", ("offset: %lu inc: %lu desired_values: %lu " "first_value: %lu", (ulong) offset, (ulong) increment, (ulong) nb_desired_values, (ulong) *first_value)); - DBUG_ASSERT(increment && nb_desired_values); + DBUG_ASSERT(increment); + DBUG_ASSERT(nb_desired_values); *first_value= 0; if (table->s->next_number_keypart) { From 9fd8f1b26406747f82913cf97034d9a4eb3d0a2e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 24 May 2020 17:30:57 +0200 Subject: [PATCH 24/37] mtr: update titlebar when the test ends, not when it starts otherwise it reaches "0 tests left" state and then waits for a few minutes for all workers to complete their tests. show failures. account for retries. --- mysql-test/lib/mtr_report.pm | 60 +++++++++++++++++++++++++++--------- mysql-test/mysql-test-run.pl | 50 +----------------------------- 2 files changed, 46 insertions(+), 64 deletions(-) diff --git a/mysql-test/lib/mtr_report.pm b/mysql-test/lib/mtr_report.pm index e553b0305b4..a58fb369a13 100644 --- a/mysql-test/lib/mtr_report.pm +++ b/mysql-test/lib/mtr_report.pm @@ -37,8 +37,20 @@ use My::Platform; use POSIX qw[ _exit ]; use IO::Handle qw[ flush ]; use mtr_results; - use Term::ANSIColor; +use English; + +my $tot_real_time= 0; +my $tests_done= 0; +my $tests_failed= 0; + +our $timestamp= 0; +our $timediff= 0; +our $name; +our $verbose; +our $verbose_restart= 0; +our $timer= 1; +our $tests_total; my %color_map = qw/pass green retry-pass green @@ -47,20 +59,39 @@ my %color_map = qw/pass green disabled bright_black skipped yellow reset reset/; -sub xterm_color { - if (-t STDOUT and defined $ENV{TERM} and $ENV{TERM} =~ /xterm/) { - syswrite STDOUT, color($color_map{$_[0]}); + +my $set_titlebar; +my $set_color= sub { }; + +if (-t STDOUT) { + if (IS_WINDOWS) { + eval { + require Win32::Console; + $set_titlebar = sub { &Win32::Console::Title($_[0]);}; + } + } elsif ($ENV{TERM} =~ /xterm/) { + $set_titlebar = sub { syswrite STDOUT, "\e]0;$_[0]\a"; }; + $set_color = sub { syswrite STDOUT, color($color_map{$_[0]}); } } } -my $tot_real_time= 0; +sub titlebar_stat($) { -our $timestamp= 0; -our $timediff= 0; -our $name; -our $verbose; -our $verbose_restart= 0; -our $timer= 1; + sub time_format($) { + sprintf '%d:%02d:%02d', $_[0]/3600, ($_[0]/60)%60, $_[0]%60; + } + + $tests_done++; + $tests_failed++ if $_[0] =~ /fail/; + $tests_total++ if $_[0] =~ /retry/; + + my $spent = time - $BASETIME; + my $left = $tests_total - $tests_done; + + &$set_titlebar(sprintf "mtr: spent %s on %d tests. %s (%d tests) left, %d failed", + time_format($spent), $tests_done, + time_format($spent/$tests_done * $left), $left, $tests_failed); +} sub report_option { my ($opt, $value)= @_; @@ -321,8 +352,6 @@ sub mtr_report_stats ($$$$) { if ( $timer ) { - use English; - mtr_report("Spent", sprintf("%.3f", $tot_real_time),"of", time - $BASETIME, "seconds executing testcases"); } @@ -620,10 +649,11 @@ sub mtr_report (@) { my @s = split /\[ (\S+) \]/, _name() . "@_\n"; if (@s > 1) { print $s[0]; - xterm_color($s[1]); + &$set_color($s[1]); print "[ $s[1] ]"; - xterm_color('reset'); + &$set_color('reset'); print $s[2]; + titlebar_stat($s[1]) if $set_titlebar; } else { print $s[0]; } diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index d229390cbb5..29294590ae3 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -369,32 +369,6 @@ my $opt_stop_keep_alive= $ENV{MTR_STOP_KEEP_ALIVE}; select(STDOUT); $| = 1; # Automatically flush STDOUT -my $set_titlebar; - - - BEGIN { - if (IS_WINDOWS) { - my $have_win32_console= 0; - eval { - require Win32::Console; - Win32::Console->import(); - $have_win32_console = 1; - }; - eval 'sub HAVE_WIN32_CONSOLE { $have_win32_console }'; - } else { - eval 'sub HAVE_WIN32_CONSOLE { 0 }'; - } -} - -if (-t STDOUT) { - if (IS_WINDOWS and HAVE_WIN32_CONSOLE) { - $set_titlebar = sub {Win32::Console::Title $_[0];}; - } elsif (defined $ENV{TERM} and $ENV{TERM} =~ /xterm/) { - $set_titlebar = sub { syswrite STDOUT, "\e];$_[0]\a"; }; - } -} - - main(); sub have_wsrep() { @@ -632,7 +606,7 @@ sub main { } ####################################################################### - my $num_tests= @$tests; + my $num_tests= $mtr_report::tests_total= @$tests; if ( $opt_parallel eq "auto" ) { # Try to find a suitable value for number of workers if (IS_WINDOWS) @@ -1073,8 +1047,6 @@ sub run_test_server ($$$) { delete $next->{reserved}; } - titlebar_stat(scalar(@$tests)) if $set_titlebar; - if ($next) { # We don't need this any more delete $next->{criteria}; @@ -6666,23 +6638,3 @@ sub list_options ($) { exit(1); } - -sub time_format($) { - sprintf '%d:%02d:%02d', $_[0]/3600, ($_[0]/60)%60, $_[0]%60; -} - -our $num_tests; - -sub titlebar_stat { - my ($left) = @_; - - # 2.5 -> best by test - $num_tests = $left + 2.5 unless $num_tests; - - my $done = $num_tests - $left; - my $spent = time - $^T; - - &$set_titlebar(sprintf "mtr: spent %s on %d tests. %s (%d tests) left", - time_format($spent), $done, - time_format($spent/$done * $left), $left); -} From 2c9c9acbfc8740dfacc2c0ce88717dfa203b479c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 26 May 2020 12:11:24 +0200 Subject: [PATCH 25/37] bintars should use bundled PCRE --- cmake/build_configurations/mysql_release.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index 1609c45a632..9d3785e152b 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -104,6 +104,7 @@ ELSEIF(DEB) SET(HAVE_EMBEDDED_PRIVILEGE_CONTROL ON) ELSE() SET(WITH_SSL bundled CACHE STRING "") + SET(WITH_PCRE bundled CACHE STRING "") SET(WITH_ZLIB bundled CACHE STRING "") SET(WITH_JEMALLOC static CACHE STRING "") ENDIF() From 8afcc37c685a0b78c3da2b8aa2e5fc70e33e9787 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 26 May 2020 14:11:41 +0200 Subject: [PATCH 26/37] update C/C --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index cdfecebc993..ce74fd0c400 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit cdfecebc9932a0dd5516c10505bfe78d79132e7d +Subproject commit ce74fd0c4009ed9f4bcbdb4a01e96c823e961dc3 From dac1280a65a97ac49c588d344a33fd7bb49c6737 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 28 May 2020 20:54:38 +0300 Subject: [PATCH 27/37] MDEV-18794 Assertion `!m_innodb' failed in ha_partition::cmp_ref upon SELECT from partitioned table System versioning assertion fix. Since DROP SYSTEM VERSIONING does not change list of dropped keys we should handle a special case. Caused by MDEV-19751. This fix deprecates MDEV-17091. --- .../versioning/r/partition_innodb.result | 12 +++++ .../suite/versioning/t/partition_innodb.test | 11 ++++ sql/sql_partition.cc | 50 ++++++++----------- 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/mysql-test/suite/versioning/r/partition_innodb.result b/mysql-test/suite/versioning/r/partition_innodb.result index 3c4adce099a..2489af91b7f 100644 --- a/mysql-test/suite/versioning/r/partition_innodb.result +++ b/mysql-test/suite/versioning/r/partition_innodb.result @@ -65,3 +65,15 @@ partition by range columns (a, row_start) ( partition p1 values less than (100, 100) ); ERROR HY000: Transaction-precise system versioned tables do not support partitioning by ROW START or ROW END +# +# MDEV-18794 Assertion `!m_innodb' failed in ha_partition::cmp_ref upon SELECT from partitioned table +# +create or replace table t1 (pk int auto_increment, i int, c char(1), primary key (pk), key(i)) +engine=innodb with system versioning partition by key() partitions 2; +insert into t1 (i, c) values (1, 'a'), (2, 'b'), (null, 'c'), (null, 'b'); +alter table t1 drop system versioning; +replace into t1 select * from t1; +select * from t1 where i > 0 or pk = 1000 limit 1; +pk i c +1 1 a +drop table t1; diff --git a/mysql-test/suite/versioning/t/partition_innodb.test b/mysql-test/suite/versioning/t/partition_innodb.test index aaaf7001644..79130dbbccf 100644 --- a/mysql-test/suite/versioning/t/partition_innodb.test +++ b/mysql-test/suite/versioning/t/partition_innodb.test @@ -78,4 +78,15 @@ partition by range columns (a, row_start) ( partition p1 values less than (100, 100) ); +--echo # +--echo # MDEV-18794 Assertion `!m_innodb' failed in ha_partition::cmp_ref upon SELECT from partitioned table +--echo # +create or replace table t1 (pk int auto_increment, i int, c char(1), primary key (pk), key(i)) +engine=innodb with system versioning partition by key() partitions 2; +insert into t1 (i, c) values (1, 'a'), (2, 'b'), (null, 'c'), (null, 'b'); +alter table t1 drop system versioning; +replace into t1 select * from t1; +select * from t1 where i > 0 or pk = 1000 limit 1; +drop table t1; + --source suite/versioning/common_finish.inc diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index b7b400f490a..f7158a2b1ee 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -5986,26 +5986,35 @@ the generated partition syntax in a correct manner. } } /* - Prohibit inplace when partitioned by primary key and the primary key is dropped. + Prohibit inplace when partitioned by primary key and the primary key is changed. */ if (!*partition_changed && tab_part_info->part_field_array && !tab_part_info->part_field_list.elements && table->s->primary_key != MAX_KEY) { - KEY *primary_key= table->key_info + table->s->primary_key; - List_iterator_fast drop_it(alter_info->drop_list); - const char *primary_name= primary_key->name.str; - const Alter_drop *drop; - drop_it.rewind(); - while ((drop= drop_it++)) + + if (alter_info->flags & (ALTER_DROP_SYSTEM_VERSIONING | + ALTER_ADD_SYSTEM_VERSIONING)) { - if (drop->type == Alter_drop::KEY && - 0 == my_strcasecmp(system_charset_info, primary_name, drop->name)) - break; + *partition_changed= true; + } + else + { + KEY *primary_key= table->key_info + table->s->primary_key; + List_iterator_fast drop_it(alter_info->drop_list); + const char *primary_name= primary_key->name.str; + const Alter_drop *drop; + drop_it.rewind(); + while ((drop= drop_it++)) + { + if (drop->type == Alter_drop::KEY && + 0 == my_strcasecmp(system_charset_info, primary_name, drop->name)) + break; + } + if (drop) + *partition_changed= TRUE; } - if (drop) - *partition_changed= TRUE; } } if (thd->work_part_info) @@ -6039,23 +6048,6 @@ the generated partition syntax in a correct manner. } } - // In case of PARTITION BY KEY(), check if primary key has changed - // System versioning also implicitly adds/removes primary key parts - if (alter_info->partition_flags == 0 && part_info->list_of_part_fields - && part_info->part_field_list.elements == 0) - { - if (alter_info->flags & (ALTER_DROP_SYSTEM_VERSIONING | - ALTER_ADD_SYSTEM_VERSIONING)) - *partition_changed= true; - - List_iterator it(alter_info->key_list); - Key *key; - while((key= it++) && !*partition_changed) - { - if (key->type == Key::PRIMARY) - *partition_changed= true; - } - } /* Set up partition default_engine_type either from the create_info or from the previus table From 3e9b96b6ff925f83427600956b86a984b726d3af Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 13 Jun 2019 19:29:02 +0300 Subject: [PATCH 28/37] MDEV-18794 append_drop_column() small refactoring Bogus if() logic inside the func. --- sql/sql_table.cc | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 2d9e07b9f3e..30b728adf69 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -7824,16 +7824,13 @@ blob_length_by_type(enum_field_types type) } -static void append_drop_column(THD *thd, bool dont, String *str, - Field *field) +static inline +void append_drop_column(THD *thd, String *str, Field *field) { - if (!dont) - { - if (str->length()) - str->append(STRING_WITH_LEN(", ")); - str->append(STRING_WITH_LEN("DROP COLUMN ")); - append_identifier(thd, str, &field->field_name); - } + if (str->length()) + str->append(STRING_WITH_LEN(", ")); + str->append(STRING_WITH_LEN("DROP COLUMN ")); + append_identifier(thd, str, &field->field_name); } @@ -8088,7 +8085,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, field->invisible < INVISIBLE_SYSTEM) { StringBuffer tmp; - append_drop_column(thd, false, &tmp, field); + append_drop_column(thd, &tmp, field); my_error(ER_MISSING, MYF(0), table->s->table_name.str, tmp.c_ptr()); goto err; } @@ -8141,10 +8138,10 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, !vers_system_invisible) { StringBuffer tmp; - append_drop_column(thd, dropped_sys_vers_fields & VERS_SYS_START_FLAG, - &tmp, table->vers_start_field()); - append_drop_column(thd, dropped_sys_vers_fields & VERS_SYS_END_FLAG, - &tmp, table->vers_end_field()); + if (!(dropped_sys_vers_fields & VERS_SYS_START_FLAG)) + append_drop_column(thd, &tmp, table->vers_start_field()); + if (!(dropped_sys_vers_fields & VERS_SYS_END_FLAG)) + append_drop_column(thd, &tmp, table->vers_end_field()); my_error(ER_MISSING, MYF(0), table->s->table_name.str, tmp.c_ptr()); goto err; } From dd9773b72380eeb98b9dabba219452f29ee5603b Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 28 May 2020 22:22:19 +0300 Subject: [PATCH 29/37] MDEV-22413 Server hangs upon UPDATE on a view reading from versioned partitioned table UPDATE gets access to history records because versioning conditions are not set for VIEW. This leads to endless loop of inserting history records when clustered index is rebuilt and ha_rnd_next() returns newly inserted history record. Return back original behavior of failing on write-locked table in historical query. 35b679b9 assumed that SELECT_LEX::lock_type influences anything, but actually at this point table is already locked. Original bug report was tempesta-tech/mariadb#102 --- .../suite/versioning/r/partition.result | 26 ++++++++++++-- mysql-test/suite/versioning/r/select.result | 3 +- mysql-test/suite/versioning/r/select2.result | 3 +- mysql-test/suite/versioning/t/partition.test | 36 ++++++++++++++++--- mysql-test/suite/versioning/t/select.test | 1 + mysql-test/suite/versioning/t/select2.test | 1 + sql/sql_select.cc | 10 ++++-- 7 files changed, 67 insertions(+), 13 deletions(-) diff --git a/mysql-test/suite/versioning/r/partition.result b/mysql-test/suite/versioning/r/partition.result index c4fedb08ad6..4d0426512e6 100644 --- a/mysql-test/suite/versioning/r/partition.result +++ b/mysql-test/suite/versioning/r/partition.result @@ -650,6 +650,26 @@ pk f update t1 set f=pk; delete from t1; drop table t1; -# Test cleanup -drop database test; -create database test; +# +# MDEV-22413 Server hangs upon UPDATE/DELETE on a view reading from versioned partitioned table +# +create or replace table t1 (f char(6)) engine innodb with system versioning; +insert into t1 values (null); +update t1 set f= 'foo'; +update t1 set f= 'bar'; +create or replace view v1 as select * from t1 for system_time all; +update v1 set f = ''; +ERROR HY000: Table 't1' was locked with a READ lock and can't be updated +create or replace table t1 (f char(6)) engine innodb with system versioning +partition by system_time limit 1 +(partition p1 history, partition p2 history, partition pn current); +insert into t1 values (null); +update t1 set f= 'foo'; +update t1 set f= 'bar'; +create or replace view v1 as select * from t1 for system_time all; +update v1 set f= ''; +ERROR HY000: Table 't1' was locked with a READ lock and can't be updated +delete from v1; +ERROR HY000: Table 't1' was locked with a READ lock and can't be updated +drop view v1; +drop table t1; diff --git a/mysql-test/suite/versioning/r/select.result b/mysql-test/suite/versioning/r/select.result index f1bdcccce59..74df243cf7c 100644 --- a/mysql-test/suite/versioning/r/select.result +++ b/mysql-test/suite/versioning/r/select.result @@ -206,8 +206,7 @@ ERROR HY000: Table `t1` is not system-versioned create or replace table t1 (x int) with system versioning; insert into t1 values (1); select * from t1 for system_time all for update; -x -1 +ERROR HY000: Table 't1' was locked with a READ lock and can't be updated create or replace table t1 (a int not null auto_increment primary key) with system versioning; select * from (t1 as t2 left join t1 as t3 using (a)) natural left join t1; a diff --git a/mysql-test/suite/versioning/r/select2.result b/mysql-test/suite/versioning/r/select2.result index bb5c82ee444..7b20d68343b 100644 --- a/mysql-test/suite/versioning/r/select2.result +++ b/mysql-test/suite/versioning/r/select2.result @@ -229,8 +229,7 @@ ERROR HY000: Table `t1` is not system-versioned create or replace table t1 (x int) with system versioning; insert into t1 values (1); select * from t1 for system_time as of now() for update; -x -1 +ERROR HY000: Table 't1' was locked with a READ lock and can't be updated create or replace table t1 (a int not null auto_increment primary key) with system versioning; select * from (t1 as t2 left join t1 as t3 using (a)) natural left join t1; a diff --git a/mysql-test/suite/versioning/t/partition.test b/mysql-test/suite/versioning/t/partition.test index 6fee6f43847..6cd88caf1a9 100644 --- a/mysql-test/suite/versioning/t/partition.test +++ b/mysql-test/suite/versioning/t/partition.test @@ -583,7 +583,6 @@ update t1 left join t2 on a > b set b= 2 order by b; # cleanup drop table t1, t2; ---source suite/versioning/common_finish.inc --echo # --echo # MDEV-17091 Assertion `old_part_id == m_last_part' failed in --echo # ha_partition::update_row or `part_id == m_last_part' in @@ -605,6 +604,35 @@ update t1 set f=pk; delete from t1; drop table t1; ---echo # Test cleanup -drop database test; -create database test; +--echo # +--echo # MDEV-22413 Server hangs upon UPDATE/DELETE on a view reading from versioned partitioned table +--echo # +create or replace table t1 (f char(6)) engine innodb with system versioning; + +insert into t1 values (null); +update t1 set f= 'foo'; +update t1 set f= 'bar'; + +create or replace view v1 as select * from t1 for system_time all; +--error ER_TABLE_NOT_LOCKED_FOR_WRITE +update v1 set f = ''; + +create or replace table t1 (f char(6)) engine innodb with system versioning +partition by system_time limit 1 +(partition p1 history, partition p2 history, partition pn current); + +insert into t1 values (null); +update t1 set f= 'foo'; +update t1 set f= 'bar'; + +create or replace view v1 as select * from t1 for system_time all; +--error ER_TABLE_NOT_LOCKED_FOR_WRITE +update v1 set f= ''; +--error ER_TABLE_NOT_LOCKED_FOR_WRITE +delete from v1; + +# cleanup +drop view v1; +drop table t1; + +--source suite/versioning/common_finish.inc diff --git a/mysql-test/suite/versioning/t/select.test b/mysql-test/suite/versioning/t/select.test index 069fe7a9a8d..16c2f34ab35 100644 --- a/mysql-test/suite/versioning/t/select.test +++ b/mysql-test/suite/versioning/t/select.test @@ -123,6 +123,7 @@ select * from t1 for system_time all; create or replace table t1 (x int) with system versioning; insert into t1 values (1); +--error ER_TABLE_NOT_LOCKED_FOR_WRITE select * from t1 for system_time all for update; create or replace table t1 (a int not null auto_increment primary key) with system versioning; diff --git a/mysql-test/suite/versioning/t/select2.test b/mysql-test/suite/versioning/t/select2.test index d1b73fa799b..fa5d52c5143 100644 --- a/mysql-test/suite/versioning/t/select2.test +++ b/mysql-test/suite/versioning/t/select2.test @@ -128,6 +128,7 @@ select * from t1 for system_time all; create or replace table t1 (x int) with system versioning; insert into t1 values (1); +--error ER_TABLE_NOT_LOCKED_FOR_WRITE select * from t1 for system_time as of now() for update; create or replace table t1 (a int not null auto_increment primary key) with system versioning; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 5694085d7da..05da4a1e750 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -832,10 +832,16 @@ int SELECT_LEX::vers_setup_conds(THD *thd, TABLE_LIST *tables) if (vers_conditions.is_set()) { + if (vers_conditions.was_set() && + table->lock_type > TL_READ_NO_INSERT && + !vers_conditions.delete_history) + { + my_error(ER_TABLE_NOT_LOCKED_FOR_WRITE, MYF(0), table->alias.str); + DBUG_RETURN(-1); + } + if (vers_conditions.type == SYSTEM_TIME_ALL) continue; - - lock_type= TL_READ; // ignore TL_WRITE, history is immutable anyway } const LEX_CSTRING *fstart= From 19da9a51ae174785806c87bcc8fa47406af9ed96 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 28 May 2020 22:22:20 +0300 Subject: [PATCH 30/37] MDEV-16937 Strict SQL with system versioned tables causes issues Respect system fields in NO_ZERO_DATE mode. This is the subject for refactoring in MDEV-19597 --- .../federated/federatedx_versioning.result | 4 ++-- .../suite/federated/federatedx_versioning.test | 2 +- mysql-test/suite/versioning/common.inc | 4 ++++ .../suite/versioning/engines.combinations | 5 +++++ mysql-test/suite/versioning/r/select.result | 6 +++--- .../suite/versioning/r/select2,trx_id.rdiff | 4 ++-- mysql-test/suite/versioning/r/select2.result | 10 +++++----- mysql-test/suite/versioning/r/sysvars.result | 4 ++-- mysql-test/suite/versioning/r/view.result | 2 +- mysql-test/suite/versioning/t/select.test | 10 +++++----- mysql-test/suite/versioning/t/select2.test | 10 +++++----- mysql-test/suite/versioning/t/sysvars.test | 4 ++-- mysql-test/suite/versioning/t/view.test | 2 +- sql/field.cc | 7 +++++++ sql/sql_table.cc | 1 + sql/table.cc | 2 +- sql/unireg.cc | 2 ++ storage/federatedx/ha_federatedx.cc | 17 +++++++++++++++++ 18 files changed, 66 insertions(+), 30 deletions(-) diff --git a/mysql-test/suite/federated/federatedx_versioning.result b/mysql-test/suite/federated/federatedx_versioning.result index c708b5f7932..d9e8a421989 100644 --- a/mysql-test/suite/federated/federatedx_versioning.result +++ b/mysql-test/suite/federated/federatedx_versioning.result @@ -9,8 +9,8 @@ show create table tf; Table Create Table tf CREATE TABLE `tf` ( `x` int(11) DEFAULT NULL, - `row_start` SYS_TYPE NOT NULL INVISIBLE DEFAULT 0, - `row_end` SYS_TYPE NOT NULL INVISIBLE DEFAULT 0 + `row_start` SYS_TYPE INVISIBLE DEFAULT '1971-01-01 00:00:00.000000', + `row_end` SYS_TYPE INVISIBLE DEFAULT '1971-01-01 00:00:00.000000' ) ENGINE=FEDERATED DEFAULT CHARSET=latin1 CONNECTION='mysql://root@127.0.0.1:MASTER_MYPORT/test/t1' # INSERT insert into t1 values (1); diff --git a/mysql-test/suite/federated/federatedx_versioning.test b/mysql-test/suite/federated/federatedx_versioning.test index 40d0bc5018a..44999522a4d 100644 --- a/mysql-test/suite/federated/federatedx_versioning.test +++ b/mysql-test/suite/federated/federatedx_versioning.test @@ -12,7 +12,7 @@ eval create or replace table t1 ( with system versioning; --replace_result $MASTER_MYPORT MASTER_MYPORT eval create or replace table tf engine=FEDERATED connection='mysql://root@127.0.0.1:$MASTER_MYPORT/test/t1'; ---replace_result $MASTER_MYPORT MASTER_MYPORT $sys_datatype_expl SYS_TYPE "'0000-00-00 00:00:00.000000'" 0 +--replace_result $MASTER_MYPORT MASTER_MYPORT $sys_datatype_expl SYS_TYPE 19710101000000 "'1971-01-01 00:00:00.000000'" " NOT NULL" "" show create table tf; --echo # INSERT insert into t1 values (1); diff --git a/mysql-test/suite/versioning/common.inc b/mysql-test/suite/versioning/common.inc index 355b571e5a0..efb081a02e4 100644 --- a/mysql-test/suite/versioning/common.inc +++ b/mysql-test/suite/versioning/common.inc @@ -50,6 +50,10 @@ if ($MTR_COMBINATION_MYISAM) { --let $MTR_COMBINATION_TIMESTAMP= 1 } +if ($MTR_COMBINATION_TRADITIONAL) +{ + --let $MTR_COMBINATION_TIMESTAMP= 1 +} if ($MTR_COMBINATION_HEAP) { --let $MTR_COMBINATION_TIMESTAMP= 1 diff --git a/mysql-test/suite/versioning/engines.combinations b/mysql-test/suite/versioning/engines.combinations index 26b5bab23f1..57e2af6cd06 100644 --- a/mysql-test/suite/versioning/engines.combinations +++ b/mysql-test/suite/versioning/engines.combinations @@ -7,5 +7,10 @@ default-storage-engine=innodb [myisam] default-storage-engine=myisam +[traditional] +default-storage-engine=myisam +sql-mode=traditional + [heap] default-storage-engine=memory + diff --git a/mysql-test/suite/versioning/r/select.result b/mysql-test/suite/versioning/r/select.result index 74df243cf7c..23c6e2d74af 100644 --- a/mysql-test/suite/versioning/r/select.result +++ b/mysql-test/suite/versioning/r/select.result @@ -43,7 +43,7 @@ ASOF_x y 7 107 8 108 9 109 -select x as FROMTO_x, y from t1 for system_time from timestamp '0-0-0 0:0:0' to timestamp @t1; +select x as FROMTO_x, y from t1 for system_time from timestamp '1970-01-01 00:00:00' to timestamp @t1; FROMTO_x y 0 100 1 101 @@ -55,7 +55,7 @@ FROMTO_x y 7 107 8 108 9 109 -select x as BETWAND_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; +select x as BETWAND_x, y from t1 for system_time between timestamp '1970-01-01 00:00:00' and timestamp @t1; BETWAND_x y 0 100 1 101 @@ -247,7 +247,7 @@ a b select * from (select * from (select * from t1 cross join t2) as tmp1) as tmp2; a b 1 2 -select * from (select * from t1 cross join t2 for system_time as of timestamp ('0-0-0')) as tmp; +select * from (select * from t1 cross join t2 for system_time as of timestamp ('1970-01-01 00:00:00')) as tmp; a b create or replace table t1(a1 int) with system versioning; create or replace table t2(a2 int) with system versioning; diff --git a/mysql-test/suite/versioning/r/select2,trx_id.rdiff b/mysql-test/suite/versioning/r/select2,trx_id.rdiff index a657b94c031..d4b35c45568 100644 --- a/mysql-test/suite/versioning/r/select2,trx_id.rdiff +++ b/mysql-test/suite/versioning/r/select2,trx_id.rdiff @@ -22,7 +22,7 @@ 7 107 8 108 9 109 --select x as FROMTO2_x, y from t1 for system_time from '0-0-0 0:0:0' to @t1; +-select x as FROMTO2_x, y from t1 for system_time from '1970-01-01 00:00' to @t1; +select x as FROMTO2_x, y from t1 for system_time from @x0 to @x1; FROMTO2_x y 0 100 @@ -31,7 +31,7 @@ 7 107 8 108 9 109 --select x as BETWAND2_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; +-select x as BETWAND2_x, y from t1 for system_time between timestamp '1970-01-01 00:00' and timestamp @t1; +select x as BETWAND2_x, y from t1 for system_time between transaction @x0 and transaction @x1; BETWAND2_x y 0 100 diff --git a/mysql-test/suite/versioning/r/select2.result b/mysql-test/suite/versioning/r/select2.result index 7b20d68343b..90607335fc1 100644 --- a/mysql-test/suite/versioning/r/select2.result +++ b/mysql-test/suite/versioning/r/select2.result @@ -44,7 +44,7 @@ ASOF_x y 7 107 8 108 9 109 -select x as FROMTO_x, y from t1 for system_time from '0-0-0 0:0:0' to timestamp @t1; +select x as FROMTO_x, y from t1 for system_time from '1970-01-01 00:00' to timestamp @t1; FROMTO_x y 0 100 1 101 @@ -56,7 +56,7 @@ FROMTO_x y 7 107 8 108 9 109 -select x as BETWAND_x, y from t1 for system_time between '0-0-0 0:0:0' and timestamp @t1; +select x as BETWAND_x, y from t1 for system_time between '1970-01-01 00:00' and timestamp @t1; BETWAND_x y 0 100 1 101 @@ -94,7 +94,7 @@ ASOF2_x y 7 107 8 108 9 109 -select x as FROMTO2_x, y from t1 for system_time from '0-0-0 0:0:0' to @t1; +select x as FROMTO2_x, y from t1 for system_time from '1970-01-01 00:00' to @t1; FROMTO2_x y 0 100 1 101 @@ -106,7 +106,7 @@ FROMTO2_x y 7 107 8 108 9 109 -select x as BETWAND2_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; +select x as BETWAND2_x, y from t1 for system_time between timestamp '1970-01-01 00:00' and timestamp @t1; BETWAND2_x y 0 100 1 101 @@ -270,7 +270,7 @@ a b select * from (select * from (select * from t1 cross join t2) as tmp1) as tmp2; a b 1 2 -select * from (select * from t1 cross join t2 for system_time as of timestamp ('0-0-0')) as tmp; +select * from (select * from t1 cross join t2 for system_time as of timestamp ('1970-01-01 00:00')) as tmp; a b create or replace table t1(a1 int) with system versioning; create or replace table t2(a2 int) with system versioning; diff --git a/mysql-test/suite/versioning/r/sysvars.result b/mysql-test/suite/versioning/r/sysvars.result index 899eda91a81..9cb4911b321 100644 --- a/mysql-test/suite/versioning/r/sysvars.result +++ b/mysql-test/suite/versioning/r/sysvars.result @@ -134,11 +134,11 @@ select * from t for system_time all; a 2 1 -select * from t for system_time from '0-0-0' to current_timestamp(6); +select * from t for system_time from '1970-01-01 00:00' to current_timestamp(6); a 2 1 -select * from t for system_time between '0-0-0' and current_timestamp(6); +select * from t for system_time between '1970-01-01 00:00' and current_timestamp(6); a 2 1 diff --git a/mysql-test/suite/versioning/r/view.result b/mysql-test/suite/versioning/r/view.result index 3b3fe580af4..e9e74899790 100644 --- a/mysql-test/suite/versioning/r/view.result +++ b/mysql-test/suite/versioning/r/view.result @@ -79,7 +79,7 @@ create or replace view vt12 as select * from t1 cross join t2; select * from vt12; a b 1 2 -create or replace view vt12 as select * from t1 for system_time as of timestamp ('0-0-0') cross join t2; +create or replace view vt12 as select * from t1 for system_time as of timestamp ('1970-01-01 00:00') cross join t2; select * from vt12; a b # VIEW improvements [tempesta-tech/mariadb#183] diff --git a/mysql-test/suite/versioning/t/select.test b/mysql-test/suite/versioning/t/select.test index 16c2f34ab35..c70f2714fff 100644 --- a/mysql-test/suite/versioning/t/select.test +++ b/mysql-test/suite/versioning/t/select.test @@ -48,8 +48,8 @@ if ($MTR_COMBINATION_TRX_ID) select x, y from t1; select x as ASOF_x, y from t1 for system_time as of timestamp @t0; -select x as FROMTO_x, y from t1 for system_time from timestamp '0-0-0 0:0:0' to timestamp @t1; -select x as BETWAND_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; +select x as FROMTO_x, y from t1 for system_time from timestamp '1970-01-01 00:00:00' to timestamp @t1; +select x as BETWAND_x, y from t1 for system_time between timestamp '1970-01-01 00:00:00' and timestamp @t1; select x as ALL_x, y from t1 for system_time all; --disable_query_log @@ -62,8 +62,8 @@ if ($MTR_COMBINATION_TRX_ID) if ($MTR_COMBINATION_TIMESTAMP) { select x as ASOF2_x, y from t1 for system_time as of @t0; - select x as FROMTO2_x, y from t1 for system_time from timestamp '0-0-0 0:0:0' to timestamp @t1; - select x as BETWAND2_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; + select x as FROMTO2_x, y from t1 for system_time from timestamp '1970-01-01 00:00:00' to timestamp @t1; + select x as BETWAND2_x, y from t1 for system_time between timestamp '1970-01-01 00:00:00' and timestamp @t1; } --enable_query_log @@ -161,7 +161,7 @@ insert into t1 values (1); insert into t2 values (2); select * from (select * from t1 cross join t2) as tmp; select * from (select * from (select * from t1 cross join t2) as tmp1) as tmp2; -select * from (select * from t1 cross join t2 for system_time as of timestamp ('0-0-0')) as tmp; +select * from (select * from t1 cross join t2 for system_time as of timestamp ('1970-01-01 00:00:00')) as tmp; create or replace table t1(a1 int) with system versioning; create or replace table t2(a2 int) with system versioning; diff --git a/mysql-test/suite/versioning/t/select2.test b/mysql-test/suite/versioning/t/select2.test index fa5d52c5143..7ede218cabf 100644 --- a/mysql-test/suite/versioning/t/select2.test +++ b/mysql-test/suite/versioning/t/select2.test @@ -36,8 +36,8 @@ if($MTR_COMBINATION_TRX_ID) { select x, y from t1; select x as ASOF_x, y from t1 for system_time as of timestamp @t0; -select x as FROMTO_x, y from t1 for system_time from '0-0-0 0:0:0' to timestamp @t1; -select x as BETWAND_x, y from t1 for system_time between '0-0-0 0:0:0' and timestamp @t1; +select x as FROMTO_x, y from t1 for system_time from '1970-01-01 00:00' to timestamp @t1; +select x as BETWAND_x, y from t1 for system_time between '1970-01-01 00:00' and timestamp @t1; select x as ALL_x, y from t1 for system_time all; if($MTR_COMBINATION_TRX_ID) { @@ -47,8 +47,8 @@ if($MTR_COMBINATION_TRX_ID) { } if(!$MTR_COMBINATION_TRX_ID) { select x as ASOF2_x, y from t1 for system_time as of @t0; - select x as FROMTO2_x, y from t1 for system_time from '0-0-0 0:0:0' to @t1; - select x as BETWAND2_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; + select x as FROMTO2_x, y from t1 for system_time from '1970-01-01 00:00' to @t1; + select x as BETWAND2_x, y from t1 for system_time between timestamp '1970-01-01 00:00' and timestamp @t1; } drop table t1; @@ -166,7 +166,7 @@ insert into t1 values (1); insert into t2 values (2); select * from (select * from t1 cross join t2) as tmp; select * from (select * from (select * from t1 cross join t2) as tmp1) as tmp2; -select * from (select * from t1 cross join t2 for system_time as of timestamp ('0-0-0')) as tmp; +select * from (select * from t1 cross join t2 for system_time as of timestamp ('1970-01-01 00:00')) as tmp; create or replace table t1(a1 int) with system versioning; create or replace table t2(a2 int) with system versioning; diff --git a/mysql-test/suite/versioning/t/sysvars.test b/mysql-test/suite/versioning/t/sysvars.test index 8c8311d66fc..1999ddd13fc 100644 --- a/mysql-test/suite/versioning/t/sysvars.test +++ b/mysql-test/suite/versioning/t/sysvars.test @@ -97,8 +97,8 @@ select * from t for system_time all; select * from t; select * from t for system_time as of timestamp current_timestamp(6); select * from t for system_time all; -select * from t for system_time from '0-0-0' to current_timestamp(6); -select * from t for system_time between '0-0-0' and current_timestamp(6); +select * from t for system_time from '1970-01-01 00:00' to current_timestamp(6); +select * from t for system_time between '1970-01-01 00:00' and current_timestamp(6); show status like "Feature_system_versioning"; diff --git a/mysql-test/suite/versioning/t/view.test b/mysql-test/suite/versioning/t/view.test index 288f1eb6e21..37676a7fb50 100644 --- a/mysql-test/suite/versioning/t/view.test +++ b/mysql-test/suite/versioning/t/view.test @@ -65,7 +65,7 @@ insert into t1 values (1); insert into t2 values (2); create or replace view vt12 as select * from t1 cross join t2; select * from vt12; -create or replace view vt12 as select * from t1 for system_time as of timestamp ('0-0-0') cross join t2; +create or replace view vt12 as select * from t1 for system_time as of timestamp ('1970-01-01 00:00') cross join t2; select * from vt12; --echo # VIEW improvements [tempesta-tech/mariadb#183] diff --git a/sql/field.cc b/sql/field.cc index a9cf4589a16..519a00677ec 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -11456,6 +11456,13 @@ bool Field::save_in_field_default_value(bool view_error_processing) { THD *thd= table->in_use; + /* + TODO: MDEV-19597 Refactor TABLE::vers_update_fields() via stored virtual columns + This condition will go away as well as other conditions with vers_sys_field(). + */ + if (vers_sys_field()) + return false; + if (unlikely(flags & NO_DEFAULT_VALUE_FLAG && real_type() != MYSQL_TYPE_ENUM)) { diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 30b728adf69..c6b22650d5c 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4195,6 +4195,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (thd->variables.sql_mode & MODE_NO_ZERO_DATE && !sql_field->default_value && !sql_field->vcol_info && + !sql_field->vers_sys_field() && sql_field->is_timestamp_type() && !opt_explicit_defaults_for_timestamp && (sql_field->flags & NOT_NULL_FLAG) && diff --git a/sql/table.cc b/sql/table.cc index c91e9d84a59..7d09985094c 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -8222,7 +8222,7 @@ bool TABLE::validate_default_values_of_unset_fields(THD *thd) const for (Field **fld= field; *fld; fld++) { if (!bitmap_is_set(write_set, (*fld)->field_index) && - !((*fld)->flags & NO_DEFAULT_VALUE_FLAG)) + !((*fld)->flags & (NO_DEFAULT_VALUE_FLAG | VERS_SYSTEM_FIELD))) { if (!(*fld)->is_null_in_record(s->default_values) && (*fld)->validate_value_in_record_with_warn(thd, s->default_values) && diff --git a/sql/unireg.cc b/sql/unireg.cc index fa30bab15a2..b7eb1cd3457 100644 --- a/sql/unireg.cc +++ b/sql/unireg.cc @@ -1086,6 +1086,7 @@ static bool make_empty_rec(THD *thd, uchar *buff, uint table_options, null_count+= field->length & 7; if (field->default_value && !field->default_value->flags && + !field->vers_sys_field() && (!(field->flags & BLOB_FLAG) || field->real_field_type() == MYSQL_TYPE_GEOMETRY)) { @@ -1108,6 +1109,7 @@ static bool make_empty_rec(THD *thd, uchar *buff, uint table_options, delete regfield; //To avoid memory leak } else if (regfield->real_type() == MYSQL_TYPE_ENUM && + !field->vers_sys_field() && (field->flags & NOT_NULL_FLAG)) { regfield->set_notnull(); diff --git a/storage/federatedx/ha_federatedx.cc b/storage/federatedx/ha_federatedx.cc index 4a717eead20..1adb7b214f0 100644 --- a/storage/federatedx/ha_federatedx.cc +++ b/storage/federatedx/ha_federatedx.cc @@ -3611,6 +3611,9 @@ int ha_federatedx::discover_assisted(handlerton *hton, THD* thd, char buf[1024]; String query(buf, sizeof(buf), cs); static LEX_CSTRING cut_clause={STRING_WITH_LEN(" WITH SYSTEM VERSIONING")}; + static LEX_CSTRING cut_start={STRING_WITH_LEN("GENERATED ALWAYS AS ROW START")}; + static LEX_CSTRING cut_end={STRING_WITH_LEN("GENERATED ALWAYS AS ROW END")}; + static LEX_CSTRING set_ts={STRING_WITH_LEN("DEFAULT TIMESTAMP'1971-01-01 00:00:00'")}; int cut_offset; MYSQL_RES *res; MYSQL_ROW rdata; @@ -3649,7 +3652,21 @@ int ha_federatedx::discover_assisted(handlerton *hton, THD* thd, cut_offset= (int)query.length() - (int)cut_clause.length; if (cut_offset > 0 && !memcmp(query.ptr() + cut_offset, cut_clause.str, cut_clause.length)) + { query.length(cut_offset); + const char *ptr= strstr(query.ptr(), cut_start.str); + if (ptr) + { + query.replace((uint32) (ptr - query.ptr()), (uint32) cut_start.length, + set_ts.str, (uint32) set_ts.length); + } + ptr= strstr(query.ptr(), cut_end.str); + if (ptr) + { + query.replace((uint32) (ptr - query.ptr()), (uint32) cut_end.length, + set_ts.str, (uint32) set_ts.length); + } + } query.append(STRING_WITH_LEN(" CONNECTION='"), cs); query.append_for_single_quote(table_s->connect_string.str, table_s->connect_string.length); From e2d7da498231a303854a45455b0b9753af54e3be Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 May 2020 01:43:21 +0200 Subject: [PATCH 31/37] Remove unused WiX source file --- win/packaging/custom_ui.wxs | 183 ------------------------------------ 1 file changed, 183 deletions(-) delete mode 100644 win/packaging/custom_ui.wxs diff --git a/win/packaging/custom_ui.wxs b/win/packaging/custom_ui.wxs deleted file mode 100644 index 70fa3ba3abd..00000000000 --- a/win/packaging/custom_ui.wxs +++ /dev/null @@ -1,183 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1 - - - - - - - 1 - - - - Create default [ProductName] instance - - - - {\WixUI_Font_Title}Default instance properties - - - - - - - Service '[SERVICENAME]' will be removed - - - - Remove default database directory '[DATABASELOCATION]' - - - - - 1 - - - 1 - - - 1 - - - - Remove default [ProductName] database - - - - {\WixUI_Font_Title}Default instance properties - - - - - - - - - - 1 - 1 - - - - WixUI_InstallMode = "Change" - !DBInstance=3 - WixUI_InstallMode = "Remove" - - - - - - - - - - - - - - - - - - SERVICENAME - - - - - - - - - - - - Running mysql_install_db.exe - - - - - - - - - - - - - - - - - - - - - - - - CMDLINE_SERVICENAME - - CMDLINE_DATABASELOCATION - - - - CMDLINE_SERVICENAME - - CMDLINE_DATABASELOCATION - - - \ No newline at end of file From ff72f36948985a0dcd2811e4d8dd66e600badc0e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 May 2020 02:06:23 +0200 Subject: [PATCH 32/37] MSI installer : Use CAQuietExec64 on Win64 , not CAQuietExec It works, but irritates people who look into the log and see traces of 32bit custom action server. --- win/packaging/create_msi.cmake.in | 2 ++ win/packaging/extra.wxs.in | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/win/packaging/create_msi.cmake.in b/win/packaging/create_msi.cmake.in index 21c59dd4f38..c589f98becc 100644 --- a/win/packaging/create_msi.cmake.in +++ b/win/packaging/create_msi.cmake.in @@ -70,10 +70,12 @@ IF(CMAKE_SIZEOF_VOID_P EQUAL 8) SET(Win64 " Win64='yes'") SET(Platform x64) SET(PlatformProgramFilesFolder ProgramFiles64Folder) + SET(CA_QUIET_EXEC CAQuietExec64) ELSE() SET(CANDLE_ARCH -arch x86) SET(Platform x86) SET(PlatformProgramFilesFolder ProgramFilesFolder) + SET(CA_QUIET_EXEC CAQuietExec) SET(Win64) ENDIF() diff --git a/win/packaging/extra.wxs.in b/win/packaging/extra.wxs.in index 3aba129fdf3..4c8e1a5a973 100644 --- a/win/packaging/extra.wxs.in +++ b/win/packaging/extra.wxs.in @@ -666,7 +666,7 @@ - From b00cd3e453f9986d0290b4a7c0ca48c509bdb0ef Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 May 2020 15:02:10 +0200 Subject: [PATCH 33/37] MDEV-22743 Windows 10 MSI installer : port in use is not determined when checking for free port, use the same logic (IPv6 socket address / dual socket), like the server would. Previous solution for testing whether port is free was trying to bind IPv4 socket on INADDR_ANY. This not work now on some reason, that attempt succeeds, even if there is an existing IPv6-dual socket listening on 0.0.0.0:3306 --- win/packaging/ca/CustomAction.cpp | 97 ++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 21 deletions(-) diff --git a/win/packaging/ca/CustomAction.cpp b/win/packaging/ca/CustomAction.cpp index 4005bd6e73b..48d56ba1183 100644 --- a/win/packaging/ca/CustomAction.cpp +++ b/win/packaging/ca/CustomAction.cpp @@ -20,6 +20,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ #undef NOMINMAX #include +#include #include #include #include @@ -33,6 +34,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ #include #include + #define ONE_MB 1048576 UINT ExecRemoveDataDirectory(wchar_t *dir) { @@ -255,36 +257,89 @@ bool ExecRemoveService(const wchar_t *name) return ret; } -/* - Check if port is free by trying to bind to the port -*/ -bool IsPortFree(short port) +/* Find whether TCP port is in use by trying to bind to the port. */ +static bool IsPortInUse(unsigned short port) { - WORD wVersionRequested; - WSADATA wsaData; + struct addrinfo* ai, * a; + struct addrinfo hints {}; - wVersionRequested = MAKEWORD(2, 2); + char port_buf[NI_MAXSERV]; + SOCKET ip_sock = INVALID_SOCKET; + hints.ai_flags = AI_PASSIVE; + hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; + snprintf(port_buf, NI_MAXSERV, "%u", (unsigned)port); - WSAStartup(wVersionRequested, &wsaData); - - struct sockaddr_in sin; - SOCKET sock; - sock = socket(AF_INET, SOCK_STREAM, 0); - if(sock == -1) + if (getaddrinfo(NULL, port_buf, &hints, &ai)) { return false; } - sin.sin_port = htons(port); - sin.sin_addr.s_addr = 0; - sin.sin_addr.s_addr = INADDR_ANY; - sin.sin_family = AF_INET; - if(bind(sock, (struct sockaddr *)&sin,sizeof(struct sockaddr_in) ) == -1) + + /* + Prefer IPv6 socket to IPv4, since we'll use IPv6 dual socket, + which coveres both IP versions. + */ + for (a = ai; a; a = a->ai_next) + { + if (a->ai_family == AF_INET6 && + (ip_sock = socket(a->ai_family, a->ai_socktype, a->ai_protocol)) != INVALID_SOCKET) + { + break; + } + } + + if (ip_sock == INVALID_SOCKET) + { + for (a = ai; a; a = a->ai_next) + { + if (ai->ai_family == AF_INET && + (ip_sock = socket(a->ai_family, a->ai_socktype, a->ai_protocol)) != INVALID_SOCKET) + { + break; + } + } + } + + if (ip_sock == INVALID_SOCKET) { return false; } - closesocket(sock); + + /* Use SO_EXCLUSIVEADDRUSE to prevent multiple binding. */ + int arg = 1; + setsockopt(ip_sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, (char*)&arg, sizeof(arg)); + + /* Allow dual socket, so that IPv4 and IPv6 are both covered.*/ + if (a->ai_family == AF_INET6) + { + arg = 0; + setsockopt(ip_sock, IPPROTO_IPV6, IPV6_V6ONLY, (char*)&arg, sizeof(arg)); + } + + bool in_use = false; + if (bind(ip_sock, a->ai_addr, a->ai_addrlen) == SOCKET_ERROR) + { + DWORD last_error = WSAGetLastError(); + in_use = (last_error == WSAEADDRINUSE || last_error == WSAEACCES); + } + + freeaddrinfo(ai); + closesocket(ip_sock); + return in_use; +} + + +/* + Check if TCP port is free +*/ +bool IsPortFree(unsigned short port) +{ + WORD wVersionRequested = MAKEWORD(2, 2); + WSADATA wsaData; + WSAStartup(wVersionRequested, &wsaData); + bool in_use = IsPortInUse(port); WSACleanup(); - return true; + return !in_use; } @@ -634,7 +689,7 @@ extern "C" UINT __stdcall CheckDatabaseProperties (MSIHANDLE hInstall) goto LExit; } - short port = (short)_wtoi(Port); + unsigned short port = (unsigned short)_wtoi(Port); if (!IsPortFree(port)) { ErrorMsg = From 069200267db4cb79bbd55d06cd7ea9e1acd86ce8 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 May 2020 15:03:28 +0200 Subject: [PATCH 34/37] Fix cmake warning - custom command succeeds without creating own OUTPUT. --- sql/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 19fdd788207..17779adcff8 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -404,6 +404,7 @@ IF(WIN32 AND MYSQLD_EXECUTABLE) COMMAND ${CMAKE_COMMAND} -E make_directory data COMMAND ${CMAKE_COMMAND} -E chdir data ${CMAKE_COMMAND} ${CONFIG_PARAM} -P ${CMAKE_CURRENT_BINARY_DIR}/create_initial_db.cmake + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ DEPENDS mysqld ) From 6e6a4227c050eefda70fd5d4f906e9274cb12f39 Mon Sep 17 00:00:00 2001 From: Kentoku SHIBA Date: Thu, 28 May 2020 10:21:46 +0900 Subject: [PATCH 35/37] Add grn_db_fin_mecab_tokenizer to finalyze mecab tokenizer --- storage/mroonga/vendor/groonga/lib/db.c | 3 ++ .../mroonga/vendor/groonga/lib/tokenizers.c | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/storage/mroonga/vendor/groonga/lib/db.c b/storage/mroonga/vendor/groonga/lib/db.c index cba22aa0e64..f3769f9aa4c 100644 --- a/storage/mroonga/vendor/groonga/lib/db.c +++ b/storage/mroonga/vendor/groonga/lib/db.c @@ -445,6 +445,9 @@ grn_db_close(grn_ctx *ctx, grn_obj *db) ctx_used_db = ctx->impl && ctx->impl->db == db; if (ctx_used_db) { +#ifdef GRN_WITH_MECAB + grn_db_fin_mecab_tokenizer(ctx); +#endif grn_ctx_loader_clear(ctx); if (ctx->impl->parser) { grn_expr_parser_close(ctx); diff --git a/storage/mroonga/vendor/groonga/lib/tokenizers.c b/storage/mroonga/vendor/groonga/lib/tokenizers.c index 11f274e72db..3daacce7ef9 100644 --- a/storage/mroonga/vendor/groonga/lib/tokenizers.c +++ b/storage/mroonga/vendor/groonga/lib/tokenizers.c @@ -797,6 +797,36 @@ grn_db_init_mecab_tokenizer(grn_ctx *ctx) } } +void +grn_db_fin_mecab_tokenizer(grn_ctx *ctx) +{ + switch (GRN_CTX_GET_ENCODING(ctx)) { + case GRN_ENC_EUC_JP : + case GRN_ENC_UTF8 : + case GRN_ENC_SJIS : +#if defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB) + { + GRN_PLUGIN_DECLARE_FUNCTIONS(tokenizers_mecab); + GRN_PLUGIN_IMPL_NAME_TAGGED(fin, tokenizers_mecab)(ctx); + } +#else /* defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB) */ + { + const char *mecab_plugin_name = "tokenizers/mecab"; + char *path; + path = grn_plugin_find_path(ctx, mecab_plugin_name); + if (path) { + GRN_FREE(path); + grn_plugin_unregister(ctx, mecab_plugin_name); + } + } +#endif /* defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB) */ + break; + default : + break; + } + return; +} + #define DEF_TOKENIZER(name, init, next, fin, vars)\ (grn_proc_create(ctx, (name), (sizeof(name) - 1),\ GRN_PROC_TOKENIZER, (init), (next), (fin), 3, (vars))) From 38ea795bb622cce6f7178291ed737ca7396a124a Mon Sep 17 00:00:00 2001 From: Kentoku SHIBA Date: Fri, 29 May 2020 01:51:30 +0900 Subject: [PATCH 36/37] Add a counter to avoid multiple initialization of Groonga mecab tokenizer --- storage/mroonga/vendor/groonga/lib/grn_tokenizers.h | 1 + .../mroonga/vendor/groonga/plugins/tokenizers/mecab.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h b/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h index e90dbfc0b31..81ac2ab6c46 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h +++ b/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h @@ -30,6 +30,7 @@ grn_rc grn_tokenizers_init(void); grn_rc grn_tokenizers_fin(void); grn_rc grn_db_init_mecab_tokenizer(grn_ctx *ctx); +void grn_db_fin_mecab_tokenizer(grn_ctx *ctx); grn_rc grn_db_init_builtin_tokenizers(grn_ctx *ctx); #ifdef __cplusplus diff --git a/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c b/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c index 3dd969a89c5..cabf2c94e53 100644 --- a/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c +++ b/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c @@ -31,6 +31,7 @@ #include #include +static unsigned int sole_mecab_init_counter = 0; static mecab_t *sole_mecab = NULL; static grn_plugin_mutex *sole_mecab_mutex = NULL; static grn_encoding sole_mecab_encoding = GRN_ENC_NONE; @@ -563,6 +564,11 @@ check_mecab_dictionary_encoding(grn_ctx *ctx) grn_rc GRN_PLUGIN_INIT(grn_ctx *ctx) { + ++sole_mecab_init_counter; + if (sole_mecab_init_counter > 1) + { + return GRN_SUCCESS; + } { char env[GRN_ENV_BUFFER_SIZE]; @@ -636,6 +642,11 @@ GRN_PLUGIN_REGISTER(grn_ctx *ctx) grn_rc GRN_PLUGIN_FIN(grn_ctx *ctx) { + --sole_mecab_init_counter; + if (sole_mecab_init_counter > 0) + { + return GRN_SUCCESS; + } if (sole_mecab) { mecab_destroy(sole_mecab); sole_mecab = NULL; From 2e1d10ecacc8ccfc0bda8a199c741ee36df981be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 30 May 2020 10:48:11 +0300 Subject: [PATCH 37/37] Add end-of-test markers to ease merges --- mysql-test/suite/versioning/r/partition.result | 1 + mysql-test/suite/versioning/r/partition_innodb.result | 1 + mysql-test/suite/versioning/t/partition.test | 2 ++ mysql-test/suite/versioning/t/partition_innodb.test | 2 ++ 4 files changed, 6 insertions(+) diff --git a/mysql-test/suite/versioning/r/partition.result b/mysql-test/suite/versioning/r/partition.result index 4d0426512e6..ab206db1213 100644 --- a/mysql-test/suite/versioning/r/partition.result +++ b/mysql-test/suite/versioning/r/partition.result @@ -673,3 +673,4 @@ delete from v1; ERROR HY000: Table 't1' was locked with a READ lock and can't be updated drop view v1; drop table t1; +# End of 10.3 tests diff --git a/mysql-test/suite/versioning/r/partition_innodb.result b/mysql-test/suite/versioning/r/partition_innodb.result index 2489af91b7f..b9870797cd2 100644 --- a/mysql-test/suite/versioning/r/partition_innodb.result +++ b/mysql-test/suite/versioning/r/partition_innodb.result @@ -77,3 +77,4 @@ select * from t1 where i > 0 or pk = 1000 limit 1; pk i c 1 1 a drop table t1; +# End of 10.3 tests diff --git a/mysql-test/suite/versioning/t/partition.test b/mysql-test/suite/versioning/t/partition.test index 6cd88caf1a9..726b991c00e 100644 --- a/mysql-test/suite/versioning/t/partition.test +++ b/mysql-test/suite/versioning/t/partition.test @@ -635,4 +635,6 @@ delete from v1; drop view v1; drop table t1; +--echo # End of 10.3 tests + --source suite/versioning/common_finish.inc diff --git a/mysql-test/suite/versioning/t/partition_innodb.test b/mysql-test/suite/versioning/t/partition_innodb.test index 79130dbbccf..29ec58af13c 100644 --- a/mysql-test/suite/versioning/t/partition_innodb.test +++ b/mysql-test/suite/versioning/t/partition_innodb.test @@ -89,4 +89,6 @@ replace into t1 select * from t1; select * from t1 where i > 0 or pk = 1000 limit 1; drop table t1; +--echo # End of 10.3 tests + --source suite/versioning/common_finish.inc