From c612a1e77c36abe08576d67de30b7e45365cad30 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 30 Jun 2018 11:02:49 +0100 Subject: [PATCH 01/29] MDEV-16596 : Windows - redo log does not work on native 4K sector disks. Disks with native 4K sectors need 4K alignment and size for unbuffered IO (i.e for files opened with FILE_FLAG_NO_BUFFERING) Innodb opens redo log with FILE_FLAG_NO_BUFFERING, however it always does 512byte IOs. Thus, the IO on 4K native sectors will fail, rendering Innodb non-functional. The fix is to check whether OS_FILE_LOG_BLOCK_SIZE is multiple of logical sector size, and if it is not, reopen the redo log without FILE_FLAG_NO_BUFFERING flag. --- cmake/os/Windows.cmake | 2 +- storage/innobase/os/os0file.cc | 107 ++++++++++++++++++++++----------- 2 files changed, 73 insertions(+), 36 deletions(-) diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index c6cbfca2346..e94099670f9 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -47,7 +47,7 @@ IF(CMAKE_C_COMPILER MATCHES "icl") ENDIF() ADD_DEFINITIONS(-D_WINDOWS -D__WIN__ -D_CRT_SECURE_NO_DEPRECATE) -ADD_DEFINITIONS(-D_WIN32_WINNT=0x0600) +ADD_DEFINITIONS(-D_WIN32_WINNT=0x0A00) # We do not want the windows.h macros min/max ADD_DEFINITIONS(-DNOMINMAX) # Speed up build process excluding unused header files diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 01b801461d6..45fb04600f3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -4109,6 +4109,32 @@ next_file: return(status); } +/** Check that IO of specific size is possible for the file +opened with FILE_FLAG_NO_BUFFERING. + +The requirement is that IO is multiple of the disk sector size. + +@param[in] file file handle +@param[in] io_size expected io size +@return true - unbuffered io of requested size is possible, false otherwise. + +@note: this function only works correctly with Windows 8 or later, +(GetFileInformationByHandleEx with FileStorageInfo is only supported there). +It will return true on earlier Windows version. + */ +static bool unbuffered_io_possible(HANDLE file, size_t io_size) +{ + FILE_STORAGE_INFO info; + if (GetFileInformationByHandleEx( + file, FileStorageInfo, &info, sizeof(info))) { + ULONG sector_size = info.LogicalBytesPerSector; + if (sector_size) + return io_size % sector_size == 0; + } + return true; +} + + /** NOTE! Use the corresponding macro os_file_create(), not directly this function! Opens an existing file or creates a new. @@ -4284,46 +4310,57 @@ os_file_create_func( access |= GENERIC_WRITE; } - do { + for (;;) { + const char *operation; + /* Use default security attributes and no template file. */ file = CreateFile( - (LPCTSTR) name, access, share_mode, NULL, + name, access, share_mode, NULL, create_flag, attributes, NULL); - if (file == INVALID_HANDLE_VALUE) { - const char* operation; - - operation = (create_mode == OS_FILE_CREATE - && !read_only) - ? "create" : "open"; - - *success = false; - - if (on_error_no_exit) { - retry = os_file_handle_error_no_exit( - name, operation, on_error_silent); - } else { - retry = os_file_handle_error(name, operation); - } - } else { - - retry = false; - - *success = true; - - if (srv_use_native_aio && ((attributes & FILE_FLAG_OVERLAPPED) != 0)) { - /* Bind the file handle to completion port. Completion port - might not be created yet, in some stages of backup, but - must always be there for the server.*/ - HANDLE port =(type == OS_LOG_FILE)? - log_completion_port : data_completion_port; - ut_a(port || srv_operation != SRV_OPERATION_NORMAL); - if (port) { - ut_a(CreateIoCompletionPort(file, port, 0, 0)); - } - } + /* If FILE_FLAG_NO_BUFFERING was set, check if this can work at all, + for expected IO sizes. Reopen without the unbuffered flag, if it is won't work*/ + if ((file != INVALID_HANDLE_VALUE) + && (attributes & FILE_FLAG_NO_BUFFERING) + && (type == OS_LOG_FILE) + && !unbuffered_io_possible(file, OS_FILE_LOG_BLOCK_SIZE)) { + ut_a(CloseHandle(file)); + attributes &= ~FILE_FLAG_NO_BUFFERING; + continue; } - } while (retry); + + *success = (file != INVALID_HANDLE_VALUE); + if (*success) { + break; + } + + operation = (create_mode == OS_FILE_CREATE && !read_only) ? + "create" : "open"; + + if (on_error_no_exit) { + retry = os_file_handle_error_no_exit( + name, operation, on_error_silent); + } + else { + retry = os_file_handle_error(name, operation); + } + + if (!retry) { + break; + } + } + + if (*success && srv_use_native_aio && (attributes & FILE_FLAG_OVERLAPPED)) { + /* Bind the file handle to completion port. Completion port + might not be created yet, in some stages of backup, but + must always be there for the server.*/ + HANDLE port = (type == OS_LOG_FILE) ? + log_completion_port : data_completion_port; + ut_a(port || srv_operation != SRV_OPERATION_NORMAL); + if (port) { + ut_a(CreateIoCompletionPort(file, port, 0, 0)); + } + } return(file); } From b71c9ae030ecafb31bc0b424a19d1354f2dd424b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 1 Jul 2018 13:55:38 +0100 Subject: [PATCH 02/29] amend fix for MDEV-16596 - do not use CREATE_NEW flag when reopening redo log file. use OPEN_ALWAYS instead, since we know file already exist. --- storage/innobase/os/os0file.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 45fb04600f3..9853263600e 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -4326,6 +4326,7 @@ os_file_create_func( && !unbuffered_io_possible(file, OS_FILE_LOG_BLOCK_SIZE)) { ut_a(CloseHandle(file)); attributes &= ~FILE_FLAG_NO_BUFFERING; + create_flag = OPEN_ALWAYS; continue; } From 8639e288086247ce39917f4cb55191c8bb5b5a8c Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Fri, 20 Apr 2018 07:06:25 +0000 Subject: [PATCH 03/29] MDEV-16630: Ambiguous error message when check constraint matches table name One can create table with the same name for `field` and `table` `check` constraint. For example: `create table t(a int check(a>0), constraint a check(a>10));` But when inserting new rows same error is always raised. For example with ```insert into t values (-1);``` and ```insert into t values (10);``` same error `ER_CONSTRAINT_FAILED` is obtained and it is not clear which constraint is violated. This patch solve this error so that in case if field constraint is violated the first parameter in the error message is `table.field_name` and if table constraint is violated the first parameter in error message is `constraint_name`. --- mysql-test/r/check_constraint.result | 37 +++++++++++++++++++++++----- mysql-test/r/constraints.result | 2 +- mysql-test/r/type_json.result | 2 +- mysql-test/t/check_constraint.test | 28 +++++++++++++++++++++ sql/table.cc | 12 ++++++++- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/mysql-test/r/check_constraint.result b/mysql-test/r/check_constraint.result index 9a32e6f12bc..9a228f9ccc7 100644 --- a/mysql-test/r/check_constraint.result +++ b/mysql-test/r/check_constraint.result @@ -10,9 +10,9 @@ t1 CREATE TABLE `t1` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 insert into t1 values (100,100); insert into t1 values (1,1); -ERROR 23000: CONSTRAINT `a` failed for `test`.`t1` +ERROR 23000: CONSTRAINT `t1.a` failed for `test`.`t1` insert into t1 values (20,1); -ERROR 23000: CONSTRAINT `b` failed for `test`.`t1` +ERROR 23000: CONSTRAINT `t1.b` failed for `test`.`t1` insert into t1 values (20,30); ERROR 23000: CONSTRAINT `min` failed for `test`.`t1` insert into t1 values (500,500); @@ -60,7 +60,7 @@ t1 CREATE TABLE `t1` ( CONSTRAINT `CONSTRAINT_1` CHECK (`a` + `b` + `c` < 500) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 insert into t1 values(105,105,105); -ERROR 23000: CONSTRAINT `c` failed for `test`.`t1` +ERROR 23000: CONSTRAINT `t1.c` failed for `test`.`t1` insert into t1 values(249,249,9); ERROR 23000: CONSTRAINT `CONSTRAINT_1` failed for `test`.`t1` insert into t1 values(105,105,9); @@ -145,7 +145,7 @@ ERROR HY000: Function or expression '@b' cannot be used in the CHECK clause of ` create table t1 (a int check (a = 1)); insert t1 values (1); insert t1 values (2); -ERROR 23000: CONSTRAINT `a` failed for `test`.`t1` +ERROR 23000: CONSTRAINT `t1.a` failed for `test`.`t1` insert t1 values (NULL); select * from t1; a @@ -165,13 +165,13 @@ ERROR 22007: Truncated incorrect DOUBLE value: 'Ken' SHOW WARNINGS; Level Code Message Error 1292 Truncated incorrect DOUBLE value: 'Ken' -Error 4025 CONSTRAINT `FirstName` failed for `test`.`t1` +Error 4025 CONSTRAINT `t1.FirstName` failed for `test`.`t1` INSERT INTO t1 VALUES (NULL, 'Ken'),(NULL, 'Brian'); ERROR 22007: Truncated incorrect DOUBLE value: 'Ken' SHOW WARNINGS; Level Code Message Error 1292 Truncated incorrect DOUBLE value: 'Ken' -Error 4025 CONSTRAINT `FirstName` failed for `test`.`t1` +Error 4025 CONSTRAINT `t1.FirstName` failed for `test`.`t1` INSERT IGNORE INTO t1 VALUES (NULL, 'Ken'); Warnings: Warning 1292 Truncated incorrect DOUBLE value: 'Ken' @@ -197,3 +197,28 @@ EmployeeID FirstName 5 Ken 6 Brian drop table t1; +# +# MDEV-16630: Ambiguous error message when check constraint +# matches table name +# +use test; +drop table if exists t; +create table t(a int, b int check(b>0), +constraint b check(a0), +constraint x check (a>10)); +show create table t; +Table Create Table +t CREATE TABLE `t` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL CHECK (`b` > 0), + CONSTRAINT `b` CHECK (`a` < `b`), + CONSTRAINT `a` CHECK (`a` > 0), + CONSTRAINT `x` CHECK (`a` > 10) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +# Field constraint 'b' will fail +insert into t values (-1, 0); +ERROR 23000: CONSTRAINT `t.b` failed for `test`.`t` +# Table constraint 'b' will fail +insert into t values (1,1); +ERROR 23000: CONSTRAINT `b` failed for `test`.`t` +drop table t; diff --git a/mysql-test/r/constraints.result b/mysql-test/r/constraints.result index fe9398ea8ce..57cfbfb3d37 100644 --- a/mysql-test/r/constraints.result +++ b/mysql-test/r/constraints.result @@ -7,7 +7,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 insert into t1 values (1); insert into t1 values (0); -ERROR 23000: CONSTRAINT `a` failed for `test`.`t1` +ERROR 23000: CONSTRAINT `t1.a` failed for `test`.`t1` drop table t1; create table t1 (a int, b int, check (a>b)); show create table t1; diff --git a/mysql-test/r/type_json.result b/mysql-test/r/type_json.result index 2c1fdbe2b95..0045847097b 100644 --- a/mysql-test/r/type_json.result +++ b/mysql-test/r/type_json.result @@ -20,7 +20,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 insert t1 values ('[]'); insert t1 values ('a'); -ERROR 23000: CONSTRAINT `a` failed for `test`.`t1` +ERROR 23000: CONSTRAINT `t1.a` failed for `test`.`t1` set timestamp=unix_timestamp('2010:11:12 13:14:15'); create or replace table t1(a json default(json_object('now', now()))); show create table t1; diff --git a/mysql-test/t/check_constraint.test b/mysql-test/t/check_constraint.test index 02081071bd4..be06c3ec579 100644 --- a/mysql-test/t/check_constraint.test +++ b/mysql-test/t/check_constraint.test @@ -135,3 +135,31 @@ INSERT INTO t1 VALUES (NULL, 'Ken'),(NULL, 'Brian'); set sql_mode=default; select * from t1; drop table t1; + +--echo # +--echo # MDEV-16630: Ambiguous error message when check constraint +--echo # matches table name +--echo # + +use test; +--disable_warnings +drop table if exists t; +--enable_warnings + +create table t(a int, b int check(b>0), + constraint b check(a0), + constraint x check (a>10)); + +show create table t; + +# Generate error when field constraint 'b' is violated +--echo # Field constraint 'b' will fail +--error ER_CONSTRAINT_FAILED +insert into t values (-1, 0); + +# Generate error when table constraint 'b' is violated. +--echo # Table constraint 'b' will fail +--error ER_CONSTRAINT_FAILED +insert into t values (1,1); + +drop table t; diff --git a/sql/table.cc b/sql/table.cc index 917d194047c..7cb84bcc5ea 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -5142,8 +5142,18 @@ int TABLE::verify_constraints(bool ignore_failure) if (((*chk)->expr->val_int() == 0 && !(*chk)->expr->null_value) || in_use->is_error()) { + StringBuffer field_error(system_charset_info); + enum_vcol_info_type vcol_type= (*chk)->get_vcol_type(); + DBUG_ASSERT(vcol_type == VCOL_CHECK_TABLE || + vcol_type == VCOL_CHECK_FIELD); + if (vcol_type == VCOL_CHECK_FIELD) + { + field_error.append(s->table_name.str); + field_error.append("."); + } + field_error.append((*chk)->name.str); my_error(ER_CONSTRAINT_FAILED, - MYF(ignore_failure ? ME_JUST_WARNING : 0), (*chk)->name.str, + MYF(ignore_failure ? ME_JUST_WARNING : 0), field_error.c_ptr(), s->db.str, s->table_name.str); return ignore_failure ? VIEW_CHECK_SKIP : VIEW_CHECK_ERROR; } From 400cf017152c732387c89deaa082b43c8fb42d71 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 2 Jul 2018 17:19:42 +0100 Subject: [PATCH 04/29] MDEV-16571 - some backup tests sometimes with missing data after restore. Marko mentions, it could be caused by MDEV-15740 where InnoDB does not flush redo log as often as it should, with innodb_flush_log_at_trx_commit=1 The workaround is to use innodb_flush_log_at_trx_commit=2, which, according to MDEV-15740 is more durable. --- mysql-test/suite/mariabackup/suite.opt | 2 +- mysql-test/suite/mariabackup/unsupported_redo.result | 3 --- mysql-test/suite/mariabackup/unsupported_redo.test | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mysql-test/suite/mariabackup/suite.opt b/mysql-test/suite/mariabackup/suite.opt index 3b5cc4f4c45..de3637814b2 100644 --- a/mysql-test/suite/mariabackup/suite.opt +++ b/mysql-test/suite/mariabackup/suite.opt @@ -1 +1 @@ ---innodb --loose-changed_page_bitmaps --innodb-sys-tables +--innodb --loose-changed_page_bitmaps --innodb-sys-tables --innodb-flush-log-at-trx-commit=2 diff --git a/mysql-test/suite/mariabackup/unsupported_redo.result b/mysql-test/suite/mariabackup/unsupported_redo.result index 543e564d8a8..a1f95c099cd 100644 --- a/mysql-test/suite/mariabackup/unsupported_redo.result +++ b/mysql-test/suite/mariabackup/unsupported_redo.result @@ -5,9 +5,6 @@ call mtr.add_suppression("InnoDB: If you are installing InnoDB, remember that yo call mtr.add_suppression("InnoDB: Ignoring tablespace for `test`\\.`t21` because it could not be opened"); call mtr.add_suppression("InnoDB: Cannot open datafile for read-only: "); call mtr.add_suppression("Table .* in the InnoDB data dictionary has tablespace id .*, but tablespace with that id or name does not exist"); -SELECT @@GLOBAL.innodb_flush_log_at_trx_commit; -@@GLOBAL.innodb_flush_log_at_trx_commit -1 CREATE TABLE t1(i INT PRIMARY KEY auto_increment, a int) ENGINE INNODB; ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; # Fails during full backup diff --git a/mysql-test/suite/mariabackup/unsupported_redo.test b/mysql-test/suite/mariabackup/unsupported_redo.test index 9d54c5bbe87..319ee2c7571 100644 --- a/mysql-test/suite/mariabackup/unsupported_redo.test +++ b/mysql-test/suite/mariabackup/unsupported_redo.test @@ -10,7 +10,6 @@ call mtr.add_suppression("Table .* in the InnoDB data dictionary has tablespace let $basedir=$MYSQLTEST_VARDIR/tmp/backup; let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; -SELECT @@GLOBAL.innodb_flush_log_at_trx_commit; CREATE TABLE t1(i INT PRIMARY KEY auto_increment, a int) ENGINE INNODB; --source ../../suite/innodb/include/no_checkpoint_start.inc ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; From 1748a31ae8d69e4939336f644f884e9de3039e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jul 2018 15:10:06 +0300 Subject: [PATCH 05/29] MDEV-16675 Unnecessary explicit lock acquisition during UPDATE or DELETE In InnoDB, an INSERT will not create an explicit lock object. Instead, the inserted record is initially implicitly locked by the transaction that wrote its trx_t::id to the hidden system column DB_TRX_ID. (Other transactions would check if DB_TRX_ID is referring to a transaction that has not been committed.) If a record was inserted in the current transaction, it would be implicitly locked by that transaction. Only if some other transaction is requesting access to the record, the implicit lock should be converted to an explicit one, so that the waits-for graph can be constructed for detecting deadlocks and lock wait timeouts. Before this fix, InnoDB would convert implicit locks to explicit ones, even if no conflict exists. lock_rec_convert_impl_to_expl(): Return whether caller_trx already holds an explicit lock that covers the record. row_vers_impl_x_locked_low(): Avoid a lookup if the record matches caller_trx->id. lock_trx_has_expl_x_lock(): Renamed from lock_trx_has_rec_x_lock(). row_upd_clust_step(): In a debug assertion, check for implicit lock before invoking lock_trx_has_expl_x_lock(). rw_trx_hash_t::find(): Make do_ref_count a mandatory parameter. Assert that trx_id is not 0 (the caller should check it). trx_sys_t::is_registered(): Only invoke find() if id != 0. trx_sys_t::find(): Add the optional parameter do_ref_count. lock_rec_queue_validate(): Avoid lookup for trx_id == 0. --- mysql-test/main/xa.result | 13 +-- mysql-test/main/xa.test | 22 ++--- .../innodb/r/innodb_information_schema.result | 2 +- mysql-test/suite/innodb/r/monitor.result | 16 ++- mysql-test/suite/innodb/t/monitor.test | 21 +++- storage/innobase/include/lock0lock.h | 16 +-- storage/innobase/include/trx0sys.h | 13 +-- storage/innobase/lock/lock0lock.cc | 99 +++++++++++++------ storage/innobase/row/row0upd.cc | 13 ++- storage/innobase/row/row0vers.cc | 22 +++-- storage/innobase/trx/trx0trx.cc | 2 +- 11 files changed, 157 insertions(+), 82 deletions(-) diff --git a/mysql-test/main/xa.result b/mysql-test/main/xa.result index 46bfa6e962a..f77c0afdec5 100644 --- a/mysql-test/main/xa.result +++ b/mysql-test/main/xa.result @@ -296,20 +296,21 @@ DROP TABLE t1; # ASSERTION THD->TRANSACTION.XID_STATE.XID.IS_NULL() # FAILED # -DROP TABLE IF EXISTS t1, t2; CREATE TABLE t1 (a INT) ENGINE=InnoDB; CREATE TABLE t2 (a INT) ENGINE=InnoDB; -START TRANSACTION; -INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (1); +COMMIT; +BEGIN; +INSERT INTO t2 VALUES (2); +UPDATE t2 SET a=a+1; connect con2,localhost,root; XA START 'xid1'; +INSERT INTO t1 VALUES (1); # Sending: -INSERT INTO t2 SELECT a FROM t1; +DELETE FROM t2; connection default; -# Waiting until INSERT ... is blocked DELETE FROM t1; connection con2; -# Reaping: INSERT INTO t2 SELECT a FROM t1 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction XA COMMIT 'xid1'; ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected diff --git a/mysql-test/main/xa.test b/mysql-test/main/xa.test index b5ef5a118b1..176ef6aa760 100644 --- a/mysql-test/main/xa.test +++ b/mysql-test/main/xa.test @@ -393,33 +393,30 @@ DROP TABLE t1; --echo # FAILED --echo # ---disable_warnings -DROP TABLE IF EXISTS t1, t2; ---enable_warnings - CREATE TABLE t1 (a INT) ENGINE=InnoDB; CREATE TABLE t2 (a INT) ENGINE=InnoDB; -START TRANSACTION; -INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (1); COMMIT; +BEGIN; +INSERT INTO t2 VALUES (2); +UPDATE t2 SET a=a+1; --connect (con2,localhost,root) XA START 'xid1'; +INSERT INTO t1 VALUES (1); --echo # Sending: ---send INSERT INTO t2 SELECT a FROM t1 +--send DELETE FROM t2 --connection default let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist - WHERE state = "Sending data" - AND info = "INSERT INTO t2 SELECT a FROM t1"; ---echo # Waiting until INSERT ... is blocked + WHERE state = "Updating" + AND info = "DELETE FROM t2"; --source include/wait_condition.inc --sleep 0.1 -DELETE FROM t1; +--send DELETE FROM t1 --connection con2 ---echo # Reaping: INSERT INTO t2 SELECT a FROM t1 --error ER_LOCK_DEADLOCK --reap --error ER_XA_RBDEADLOCK @@ -427,6 +424,7 @@ XA COMMIT 'xid1'; connection default; +reap; COMMIT; connection con2; diff --git a/mysql-test/suite/innodb/r/innodb_information_schema.result b/mysql-test/suite/innodb/r/innodb_information_schema.result index c1625f2bc3c..766d5f47c2d 100644 --- a/mysql-test/suite/innodb/r/innodb_information_schema.result +++ b/mysql-test/suite/innodb/r/innodb_information_schema.result @@ -45,7 +45,7 @@ trx_last_foreign_key_error varchar(256) YES NULL trx_is_read_only int(1) NO 0 trx_autocommit_non_locking int(1) NO 0 trx_state trx_weight trx_tables_in_use trx_tables_locked trx_rows_locked trx_rows_modified trx_concurrency_tickets trx_isolation_level trx_unique_checks trx_foreign_key_checks -RUNNING 4 0 1 7 1 0 REPEATABLE READ 1 1 +RUNNING 3 0 1 5 1 0 REPEATABLE READ 1 1 trx_isolation_level trx_unique_checks trx_foreign_key_checks SERIALIZABLE 0 0 trx_state trx_isolation_level trx_last_foreign_key_error diff --git a/mysql-test/suite/innodb/r/monitor.result b/mysql-test/suite/innodb/r/monitor.result index 2700479e7f7..a40bfdac0d0 100644 --- a/mysql-test/suite/innodb/r/monitor.result +++ b/mysql-test/suite/innodb/r/monitor.result @@ -662,7 +662,21 @@ SELECT NAME, COUNT > 0 FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE NAME LIKE 'buffer_page_written_index_leaf'; NAME COUNT > 0 buffer_page_written_index_leaf 1 +DROP TABLE t1; +CREATE TABLE t1(id INT PRIMARY KEY, a INT, b CHAR(1), UNIQUE KEY u(a,b)) +ENGINE=InnoDB; +SET @start = (SELECT COUNT FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE NAME += 'lock_rec_lock_created'); +BEGIN; +INSERT INTO t1 VALUES(1,1,'a'),(2,9999,'b'),(3,10000,'c'),(4,4,'d'); +DELETE FROM t1 WHERE a = 9999 AND b='b'; +COMMIT; +SET @end = (SELECT COUNT FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE NAME += 'lock_rec_lock_created'); +SELECT @end - @start; +@end - @start +0 +DROP TABLE t1; SET GLOBAL innodb_monitor_enable=default; SET GLOBAL innodb_monitor_disable=default; SET GLOBAL innodb_monitor_reset_all=default; -DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/monitor.test b/mysql-test/suite/innodb/t/monitor.test index dfae93694bf..3535c9c85ad 100644 --- a/mysql-test/suite/innodb/t/monitor.test +++ b/mysql-test/suite/innodb/t/monitor.test @@ -422,10 +422,27 @@ UNLOCK TABLES; SELECT NAME, COUNT > 0 FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE NAME LIKE 'buffer_page_written_index_leaf'; +DROP TABLE t1; + +CREATE TABLE t1(id INT PRIMARY KEY, a INT, b CHAR(1), UNIQUE KEY u(a,b)) +ENGINE=InnoDB; + +SET @start = (SELECT COUNT FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE NAME += 'lock_rec_lock_created'); + +BEGIN; +INSERT INTO t1 VALUES(1,1,'a'),(2,9999,'b'),(3,10000,'c'),(4,4,'d'); +DELETE FROM t1 WHERE a = 9999 AND b='b'; +COMMIT; + +SET @end = (SELECT COUNT FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE NAME += 'lock_rec_lock_created'); +SELECT @end - @start; + +DROP TABLE t1; + --disable_warnings SET GLOBAL innodb_monitor_enable=default; SET GLOBAL innodb_monitor_disable=default; SET GLOBAL innodb_monitor_reset_all=default; --enable_warnings - -DROP TABLE t1; diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index 2159f56d018..e4b8947ac3a 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -789,19 +789,21 @@ const lock_t* lock_trx_has_sys_table_locks( /*=========================*/ const trx_t* trx) /*!< in: transaction to check */ - MY_ATTRIBUTE((warn_unused_result)); + MY_ATTRIBUTE((nonnull, warn_unused_result)); -/*******************************************************************//** -Check if the transaction holds an exclusive lock on a record. -@return whether the locks are held */ +/** Check if the transaction holds an explicit exclusive lock on a record. +@param[in] trx transaction +@param[in] table table +@param[in] block leaf page +@param[in] heap_no heap number identifying the record +@return whether an explicit X-lock is held */ bool -lock_trx_has_rec_x_lock( -/*====================*/ +lock_trx_has_expl_x_lock( const trx_t* trx, /*!< in: transaction to check */ const dict_table_t* table, /*!< in: table to check */ const buf_block_t* block, /*!< in: buffer block of the record */ ulint heap_no)/*!< in: record heap number */ - MY_ATTRIBUTE((warn_unused_result)); + MY_ATTRIBUTE((nonnull, warn_unused_result)); #endif /* UNIV_DEBUG */ /** diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index ea01d698b3b..69374ab3fba 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -616,7 +616,7 @@ public: @retval pointer to trx */ - trx_t *find(trx_t *caller_trx, trx_id_t trx_id, bool do_ref_count= false) + trx_t *find(trx_t *caller_trx, trx_id_t trx_id, bool do_ref_count) { /* In MariaDB 10.3, purge will reset DB_TRX_ID to 0 @@ -624,9 +624,10 @@ public: always have a nonzero trx_t::id; there the value 0 is reserved for transactions that did not write or lock anything yet. + + The caller should already have handled trx_id==0 specially. */ - if (!trx_id) - return NULL; + ut_ad(trx_id); if (caller_trx && caller_trx->id == trx_id) { if (do_ref_count) @@ -1044,13 +1045,13 @@ public: bool is_registered(trx_t *caller_trx, trx_id_t id) { - return rw_trx_hash.find(caller_trx, id); + return id && find(caller_trx, id, false); } - trx_t *find(trx_t *caller_trx, trx_id_t id) + trx_t *find(trx_t *caller_trx, trx_id_t id, bool do_ref_count= true) { - return rw_trx_hash.find(caller_trx, id, true); + return rw_trx_hash.find(caller_trx, id, do_ref_count); } diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 4987d60dd5a..cd65a9ac8e0 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4937,8 +4937,12 @@ lock_rec_queue_validate( /* Unlike the non-debug code, this invariant can only succeed if the check and assertion are covered by the lock mutex. */ - const trx_t *impl_trx = trx_sys.rw_trx_hash.find(current_trx(), - lock_clust_rec_some_has_impl(rec, index, offsets)); + const trx_id_t impl_trx_id = lock_clust_rec_some_has_impl( + rec, index, offsets); + + const trx_t *impl_trx = impl_trx_id + ? trx_sys.find(current_trx(), impl_trx_id, false) + : 0; ut_ad(lock_mutex_own()); /* impl_trx cannot be committed until lock_mutex_exit() @@ -5547,18 +5551,31 @@ static void lock_rec_other_trx_holds_expl(trx_t *caller_trx, trx_t *trx, #endif /* UNIV_DEBUG */ -/*********************************************************************//** -If a transaction has an implicit x-lock on a record, but no explicit x-lock -set on the record, sets one for it. */ +/** If an implicit x-lock exists on a record, convert it to an explicit one. + +Often, this is called by a transaction that is about to enter a lock wait +due to the lock conflict. Two explicit locks would be created: first the +exclusive lock on behalf of the lock-holder transaction in this function, +and then a wait request on behalf of caller_trx, in the calling function. + +This may also be called by the same transaction that is already holding +an implicit exclusive lock on the record. In this case, no explicit lock +should be created. + +@param[in,out] caller_trx current transaction +@param[in] block index tree leaf page +@param[in] rec record on the leaf page +@param[in] index the index of the record +@param[in] offsets rec_get_offsets(rec,index) +@return whether caller_trx already holds an exclusive lock on rec */ static -void +bool lock_rec_convert_impl_to_expl( -/*==========================*/ - trx_t* caller_trx,/*!id)) { + return true; + } + trx = trx_sys.find(caller_trx, trx_id); } else { ut_ad(!dict_index_is_online_ddl(index)); trx = lock_sec_rec_some_has_impl(caller_trx, rec, index, offsets); + if (trx == caller_trx) { + trx->release_reference(); + return true; + } ut_d(lock_rec_other_trx_holds_expl(caller_trx, trx, rec, block)); @@ -5597,6 +5625,8 @@ lock_rec_convert_impl_to_expl( lock_rec_convert_impl_to_expl_for_trx( block, rec, index, trx, heap_no); } + + return false; } /*********************************************************************//** @@ -5641,8 +5671,11 @@ lock_clust_rec_modify_check_and_lock( /* If a transaction has no explicit x-lock set on the record, set one for it */ - lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, index, - offsets); + if (lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, index, + offsets)) { + /* We already hold an implicit exclusive lock. */ + return DB_SUCCESS; + } err = lock_rec_lock(TRUE, LOCK_X | LOCK_REC_NOT_GAP, block, heap_no, index, thr); @@ -5786,10 +5819,11 @@ lock_sec_rec_read_check_and_lock( database recovery is running. */ if (!page_rec_is_supremum(rec) - && page_get_max_trx_id(block->frame) >= trx_sys.get_min_trx_id()) { - - lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, - index, offsets); + && page_get_max_trx_id(block->frame) >= trx_sys.get_min_trx_id() + && lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, + index, offsets)) { + /* We already hold an implicit exclusive lock. */ + return DB_SUCCESS; } err = lock_rec_lock(FALSE, ulint(mode) | gap_mode, @@ -5850,10 +5884,11 @@ lock_clust_rec_read_check_and_lock( heap_no = page_rec_get_heap_no(rec); - if (heap_no != PAGE_HEAP_NO_SUPREMUM) { - - lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, - index, offsets); + if (heap_no != PAGE_HEAP_NO_SUPREMUM + && lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, + index, offsets)) { + /* We already hold an implicit exclusive lock. */ + return DB_SUCCESS; } err = lock_rec_lock(FALSE, ulint(mode) | gap_mode, @@ -6560,12 +6595,14 @@ lock_trx_has_sys_table_locks( return(strongest_lock); } -/*******************************************************************//** -Check if the transaction holds an exclusive lock on a record. -@return whether the locks are held */ +/** Check if the transaction holds an explicit exclusive lock on a record. +@param[in] trx transaction +@param[in] table table +@param[in] block leaf page +@param[in] heap_no heap number identifying the record +@return whether an explicit X-lock is held */ bool -lock_trx_has_rec_x_lock( -/*====================*/ +lock_trx_has_expl_x_lock( const trx_t* trx, /*!< in: transaction to check */ const dict_table_t* table, /*!< in: table to check */ const buf_block_t* block, /*!< in: buffer block of the record */ @@ -6574,11 +6611,9 @@ lock_trx_has_rec_x_lock( ut_ad(heap_no > PAGE_HEAP_NO_SUPREMUM); lock_mutex_enter(); - ut_a(lock_table_has(trx, table, LOCK_IX) - || table->is_temporary()); - ut_a(lock_rec_has_expl(LOCK_X | LOCK_REC_NOT_GAP, - block, heap_no, trx) - || table->is_temporary()); + ut_ad(lock_table_has(trx, table, LOCK_IX)); + ut_ad(lock_rec_has_expl(LOCK_X | LOCK_REC_NOT_GAP, block, heap_no, + trx)); lock_mutex_exit(); return(true); } diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 39206e66d75..447132dcf86 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -3093,9 +3093,7 @@ row_upd_clust_step( ulint mode; - DEBUG_SYNC_C_IF_THD( - thr_get_trx(thr)->mysql_thd, - "innodb_row_upd_clust_step_enter"); + DEBUG_SYNC_C_IF_THD(trx->mysql_thd, "innodb_row_upd_clust_step_enter"); if (dict_index_is_online_ddl(index)) { ut_ad(node->table->id != DICT_INDEXES_ID); @@ -3157,10 +3155,11 @@ row_upd_clust_step( } } - ut_ad(index->table->no_rollback() - || lock_trx_has_rec_x_lock(thr_get_trx(thr), index->table, - btr_pcur_get_block(pcur), - page_rec_get_heap_no(rec))); + ut_ad(index->table->no_rollback() || index->table->is_temporary() + || row_get_rec_trx_id(rec, index, offsets) == trx->id + || lock_trx_has_expl_x_lock(trx, index->table, + btr_pcur_get_block(pcur), + page_rec_get_heap_no(rec))); /* NOTE: the following function calls will also commit mtr */ diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index bfaa2721746..81f9c795a7c 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -126,14 +126,22 @@ row_vers_impl_x_locked_low( DBUG_RETURN(0); } - trx_t* trx = trx_sys.find(caller_trx, trx_id); + trx_t* trx; - if (trx == 0) { - /* The transaction that modified or inserted clust_rec is no - longer active, or it is corrupt: no implicit lock on rec */ - lock_check_trx_id_sanity(trx_id, clust_rec, clust_index, clust_offsets); - mem_heap_free(heap); - DBUG_RETURN(0); + if (trx_id == caller_trx->id) { + trx = caller_trx; + trx->reference(); + } else { + trx = trx_sys.find(caller_trx, trx_id); + if (trx == 0) { + /* The transaction that modified or inserted + clust_rec is no longer active, or it is + corrupt: no implicit lock on rec */ + lock_check_trx_id_sanity(trx_id, clust_rec, + clust_index, clust_offsets); + mem_heap_free(heap); + DBUG_RETURN(0); + } } comp = page_rec_is_comp(rec); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 11d83c96a92..c13ccdf53ed 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -761,7 +761,7 @@ trx_lists_init_at_db_start() for (undo = UT_LIST_GET_FIRST(rseg->undo_list); undo != NULL; undo = UT_LIST_GET_NEXT(undo_list, undo)) { - trx_t *trx = trx_sys.rw_trx_hash.find(0, undo->trx_id); + trx_t *trx = trx_sys.find(0, undo->trx_id, false); if (!trx) { trx_resurrect(undo, rseg, start_time, &rows_to_undo, false); From 058554027f6696775ca1b289688956606f59ce7d Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 5 Jul 2018 00:06:39 -0700 Subject: [PATCH 06/29] MDEV-16629 "Table Does Not Exist" Error from Recursive CTE Query Inside Function When processing a query containing with clauses a call of the function check_dependencies_in_with_clauses() before opening tables used in the query is necessary if with clauses include specifications of recursive CTEs. This call was missing if such a query belonged to a stored function. This caused misbehavior of the server: it could report a fake error as in the test case for MDEV-16629 or the executed query could hang as in the test cases for MDEV-16661 and MDEV-15151. --- mysql-test/r/cte_recursive.result | 58 +++++++++++++++++++++++++ mysql-test/t/cte_recursive.test | 72 +++++++++++++++++++++++++++++++ sql/sp_head.cc | 4 +- 3 files changed, 133 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index 2e93e9f6ff8..856f00abe59 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -3263,3 +3263,61 @@ select 3, 0*(@d:=@d+1) from qn where @d<1 select * from qn; ERROR 42000: This version of MariaDB doesn't yet support 'mix of ALL and DISTINCT UNION operations in recursive CTE spec' drop table t1; +# +# MDEV-16629: function with recursive CTE using a base table +# +CREATE TABLE t1 (id int); +INSERT INTO t1 VALUES (0), (1),(2); +WITH recursive cte AS +(SELECT id FROM t1 UNION SELECT 3 FROM cte) +SELECT count(id) FROM cte; +count(id) +4 +CREATE OR REPLACE FUNCTION func() RETURNS int +RETURN +( +WITH recursive cte AS +(SELECT id FROM t1 UNION SELECT 3 FROM cte) +SELECT count(id) FROM cte +); +SELECT func(); +func() +4 +DROP FUNCTION func; +DROP TABLE t1; +# +# MDEV-16661: function with recursive CTE using no base tables +# (fixed by the patch for MDEV-16629) +# +CREATE OR REPLACE FUNCTION func() RETURNS int +RETURN +( +WITH RECURSIVE cte AS +(SELECT 1 as id UNION SELECT * FROM cte) +SELECT count(id) FROM cte +); +SELECT func(); +func() +1 +DROP FUNCTION func; +# +# MDEV-15151: function with recursive CTE using no base tables +# (duplicate of MDEV-16661) +# +connection default; +CREATE TABLE t1 (id int KEY); +INSERT INTO t1 VALUES (0), (1),(2); +CREATE OR REPLACE FUNCTION func() RETURNS int +RETURN +( +WITH recursive cte AS +(SELECT 1 a UNION SELECT cte.* FROM cte natural join t1) +SELECT * FROM cte limit 1 +); +SELECT func();; +connect con1,localhost,root,,; +KILL QUERY 4; +DROP FUNCTION func; +DROP TABLE t1; +disconnect con1; +connection default; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 32a82c494e0..504b2c64442 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -2282,3 +2282,75 @@ select 3, 0*(@d:=@d+1) from qn where @d<1 select * from qn; drop table t1; + +--echo # +--echo # MDEV-16629: function with recursive CTE using a base table +--echo # + +CREATE TABLE t1 (id int); +INSERT INTO t1 VALUES (0), (1),(2); + +WITH recursive cte AS +(SELECT id FROM t1 UNION SELECT 3 FROM cte) +SELECT count(id) FROM cte; + +CREATE OR REPLACE FUNCTION func() RETURNS int +RETURN +( + WITH recursive cte AS + (SELECT id FROM t1 UNION SELECT 3 FROM cte) + SELECT count(id) FROM cte +); + +SELECT func(); + +DROP FUNCTION func; +DROP TABLE t1; + +--echo # +--echo # MDEV-16661: function with recursive CTE using no base tables +--echo # (fixed by the patch for MDEV-16629) +--echo # + +CREATE OR REPLACE FUNCTION func() RETURNS int +RETURN +( + WITH RECURSIVE cte AS + (SELECT 1 as id UNION SELECT * FROM cte) + SELECT count(id) FROM cte +); + +SELECT func(); + +DROP FUNCTION func; + +--echo # +--echo # MDEV-15151: function with recursive CTE using no base tables +--echo # (duplicate of MDEV-16661) +--echo # + +--connection default + +CREATE TABLE t1 (id int KEY); +INSERT INTO t1 VALUES (0), (1),(2); + +CREATE OR REPLACE FUNCTION func() RETURNS int +RETURN +( + WITH recursive cte AS + (SELECT 1 a UNION SELECT cte.* FROM cte natural join t1) + SELECT * FROM cte limit 1 +); + +--let $conid= `SELECT CONNECTION_ID()` +--send SELECT func(); + +--connect (con1,localhost,root,,) +--eval KILL QUERY $conid +--source include/restart_mysqld.inc + +DROP FUNCTION func; +DROP TABLE t1; +--disconnect con1 + +--connection default diff --git a/sql/sp_head.cc b/sql/sp_head.cc index a832aa91004..effe9d1735f 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -29,6 +29,7 @@ #include "sql_array.h" // Dynamic_array #include "log_event.h" // Query_log_event #include "sql_derived.h" // mysql_handle_derived +#include "sql_cte.h" #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation @@ -3000,7 +3001,8 @@ sp_lex_keeper::reset_lex_and_exec_core(THD *thd, uint *nextp, #endif if (open_tables) - res= instr->exec_open_and_lock_tables(thd, m_lex->query_tables); + res= check_dependencies_in_with_clauses(m_lex->with_clauses_list) || + instr->exec_open_and_lock_tables(thd, m_lex->query_tables); if (!res) { From e9f1d8da57ba9329289c3448223cc86a13a5d918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jul 2018 11:41:55 +0300 Subject: [PATCH 07/29] Fix warnings about possibly uninitialized variables --- storage/innobase/buf/buf0flu.cc | 3 +-- storage/innobase/dict/dict0dict.cc | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 99a36fc3c45..cf53a10fa21 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -965,7 +965,7 @@ buf_flush_init_for_writing( } } - uint32_t checksum; + uint32_t checksum = BUF_NO_CHECKSUM_MAGIC; switch (srv_checksum_algorithm_t(srv_checksum_algorithm)) { case SRV_CHECKSUM_ALGORITHM_INNODB: @@ -988,7 +988,6 @@ buf_flush_init_for_writing( break; case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: - checksum = BUF_NO_CHECKSUM_MAGIC; mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); break; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index df5c678ffb5..c4b88cbea56 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -6981,6 +6981,7 @@ dict_foreign_qualify_index( } if (dict_col_is_virtual(field->col)) { + col_name = ""; for (ulint j = 0; j < table->n_v_def; j++) { col_name = dict_table_get_v_col_name(table, j); if (innobase_strcasecmp(field->name,col_name) == 0) { From fdb9e66feea4ee3a8769bff74277f76e49734fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jul 2018 13:15:35 +0300 Subject: [PATCH 08/29] Implement a parameter for wait_all_purged.inc --- mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test | 4 +++- mysql-test/suite/innodb/include/wait_all_purged.inc | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test b/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test index ad733eee3a7..7feeee768ff 100644 --- a/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test +++ b/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test @@ -124,7 +124,7 @@ SELECT * FROM t1; DROP TABLE t1; -# Test adding virutal index on existing virtual column +# Test adding index on existing virtual column CREATE TABLE t1 (a INT, b INT, c INT GENERATED ALWAYS AS(a+b)); INSERT INTO t1(a, b) VALUES (1, 1), (2, 2), (3, 3), (4, 4); @@ -156,7 +156,9 @@ INSERT INTO t1(a, b) VALUES (8, 8); COMMIT; --echo # wait for purge to process the deleted/updated records. +let $wait_all_purged=1; --source ../../innodb/include/wait_all_purged.inc +let $wait_all_purged=0; SET DEBUG_SYNC= 'now SIGNAL purged'; diff --git a/mysql-test/suite/innodb/include/wait_all_purged.inc b/mysql-test/suite/innodb/include/wait_all_purged.inc index 7dbb59a5d32..c7a16888829 100644 --- a/mysql-test/suite/innodb/include/wait_all_purged.inc +++ b/mysql-test/suite/innodb/include/wait_all_purged.inc @@ -1,12 +1,18 @@ # Wait for everything to be purged. # The user should have set innodb_purge_rseg_truncate_frequency=1. +if (!$wait_all_purged) +{ + let $wait_all_purged= 0; +} +let $remaining_expect= `select concat('InnoDB ',$wait_all_purged)`; + let $wait_counter= 300; while ($wait_counter) { --replace_regex /.*History list length ([0-9]+).*/\1/ let $remaining= `SHOW ENGINE INNODB STATUS`; - if ($remaining == 'InnoDB 0') + if ($remaining == $remaining_expect) { let $wait_counter= 0; } From d897b4dc253d00f14c9b0b3bb94a99aadb358f6c Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 5 Jul 2018 00:07:55 +0530 Subject: [PATCH 09/29] MDEV-15855: Use atomics for dict_table_t::n_ref_count Make dict_table_t::n_ref_count private, and protect it with a combination of dict_sys->mutex and atomics. We want to be able to invoke dict_table_t::release() without holding dict_sys->mutex. --- storage/innobase/btr/btr0sea.cc | 2 +- storage/innobase/dict/dict0dict.cc | 4 ++-- storage/innobase/handler/ha_innodb.cc | 4 ++-- storage/innobase/include/dict0dict.ic | 19 +++++-------------- storage/innobase/include/dict0mem.h | 10 ++++++---- storage/innobase/lock/lock0lock.cc | 4 ++-- 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 278062afa78..7419ed2bfd5 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1337,7 +1337,7 @@ void btr_search_drop_page_hash_when_freed(const page_id_t& page_id) /* In all our callers, the table handle should be open, or we should be in the process of dropping the table (preventing eviction). */ - ut_ad(index->table->n_ref_count > 0 + ut_ad(index->table->get_ref_count() > 0 || mutex_own(&dict_sys->mutex)); btr_search_drop_page_hash_index(block); } diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index c4b88cbea56..8dc29863367 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -423,7 +423,7 @@ dict_table_try_drop_aborted( dict_table_t* table, /*!< in: table, or NULL if it needs to be looked up again */ table_id_t table_id, /*!< in: table identifier */ - ulint ref_count) /*!< in: expected table->n_ref_count */ + int32 ref_count) /*!< in: expected table->n_ref_count */ { trx_t* trx; @@ -1353,7 +1353,7 @@ static ibool dict_table_can_be_evicted( /*======================*/ - const dict_table_t* table) /*!< in: table to test */ + dict_table_t* table) /*!< in: table to test */ { ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_X)); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 198255a30ef..7cba4ef5794 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3058,7 +3058,7 @@ ha_innobase::update_thd( m_user_thd, thd)); /* The table should have been opened in ha_innobase::open(). */ - DBUG_ASSERT(m_prebuilt->table->n_ref_count > 0); + DBUG_ASSERT(m_prebuilt->table->get_ref_count() > 0); trx_t* trx = check_trx_exists(thd); @@ -14186,7 +14186,7 @@ ha_innobase::info_low( m_prebuilt->trx->op_info = "returning various info to MariaDB"; ib_table = m_prebuilt->table; - DBUG_ASSERT(ib_table->n_ref_count > 0); + DBUG_ASSERT(ib_table->get_ref_count() > 0); if (flag & HA_STATUS_TIME) { if (is_analyze || innobase_stats_on_metadata) { diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index ca2ea769612..fe2f8e32b1a 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1498,23 +1498,13 @@ dict_table_is_file_per_table( return !is_system_tablespace(table->space); } -/** Get reference count. -@return current value of n_ref_count */ -inline -ulint -dict_table_t::get_ref_count() const -{ - ut_ad(mutex_own(&dict_sys->mutex)); - return(n_ref_count); -} - /** Acquire the table handle. */ inline void dict_table_t::acquire() { ut_ad(mutex_own(&dict_sys->mutex)); - ++n_ref_count; + my_atomic_add32_explicit(&n_ref_count, 1, MY_MEMORY_ORDER_RELAXED); } /** Release the table handle. @@ -1523,9 +1513,10 @@ inline bool dict_table_t::release() { - ut_ad(mutex_own(&dict_sys->mutex)); - ut_ad(n_ref_count > 0); - return !--n_ref_count; + int32 n = my_atomic_add32_explicit( + &n_ref_count, -1, MY_MEMORY_ORDER_RELAXED); + ut_ad(n > 0); + return n == 1; } /** Encode the number of columns and number of virtual columns in a diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 56f695f305d..02561176706 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1357,7 +1357,11 @@ struct dict_table_t { /** Get reference count. @return current value of n_ref_count */ - inline ulint get_ref_count() const; + inline int32 get_ref_count() + { + return my_atomic_load32_explicit(&n_ref_count, + MY_MEMORY_ORDER_RELAXED); + } /** Acquire the table handle. */ inline void acquire(); @@ -1737,13 +1741,11 @@ struct dict_table_t { It is protected by lock_sys->mutex. */ ulint n_rec_locks; -#ifndef UNIV_DEBUG private: -#endif /** Count of how many handles are opened to this table. Dropping of the table is NOT allowed until this count gets to zero. MySQL does NOT itself check the number of open handles at DROP. */ - ulint n_ref_count; + int32 n_ref_count; public: /** List of locks on the table. Protected by lock_sys->mutex. */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index ee0dc5a9000..1ce798fcd43 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1493,7 +1493,7 @@ lock_rec_create_low( lock_rec_bitmap_reset(lock); lock_rec_set_nth_bit(lock, heap_no); index->table->n_rec_locks++; - ut_ad(index->table->n_ref_count > 0 || !index->table->can_be_evicted); + ut_ad(index->table->get_ref_count() > 0 || !index->table->can_be_evicted); #ifdef WITH_WSREP if (c_lock && wsrep_on_trx(trx) @@ -3664,7 +3664,7 @@ lock_table_create( lock->un_member.tab_lock.table = table; - ut_ad(table->n_ref_count > 0 || !table->can_be_evicted); + ut_ad(table->get_ref_count() > 0 || !table->can_be_evicted); UT_LIST_ADD_LAST(trx->lock.trx_locks, lock); From 1b335a74b4ea1944d6ef91113b1a510c0f11c557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jul 2018 16:44:12 +0300 Subject: [PATCH 10/29] Clean up a test At the end of a test, 'connection default' should be in a usable state. This was not the case, because there was a preceding 'send' without a 'reap'. If 'reap' was added, an error would be reported because the server was restarted after the 'send'. It is easiest to 'send' from a separate connection and do the restart from 'connection default'. --- mysql-test/r/cte_recursive.result | 6 +++--- mysql-test/t/cte_recursive.test | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index 856f00abe59..d715fa20a6f 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -3314,10 +3314,10 @@ WITH recursive cte AS (SELECT 1 a UNION SELECT cte.* FROM cte natural join t1) SELECT * FROM cte limit 1 ); -SELECT func();; connect con1,localhost,root,,; -KILL QUERY 4; +SELECT func(); +connection default; +KILL QUERY 5; DROP FUNCTION func; DROP TABLE t1; disconnect con1; -connection default; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 504b2c64442..227ccd0cdb3 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1,3 +1,4 @@ +--source include/not_embedded.inc create table t1 (a int, b varchar(32)); insert into t1 values (4,'aaaa' ), (7,'bb'), (1,'ccc'), (4,'dd'); @@ -2342,15 +2343,14 @@ RETURN SELECT * FROM cte limit 1 ); ---let $conid= `SELECT CONNECTION_ID()` ---send SELECT func(); - --connect (con1,localhost,root,,) +--let $conid= `SELECT CONNECTION_ID()` +--send SELECT func() + +--connection default --eval KILL QUERY $conid --source include/restart_mysqld.inc DROP FUNCTION func; DROP TABLE t1; --disconnect con1 - ---connection default From e9b78a1055c2b21fac1fc43dc8d660778d493fa6 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 5 Jul 2018 15:10:47 -0700 Subject: [PATCH 11/29] Removed the test case for MDEV-15151 for the following reasons: 1. The changed variant did not fail without the patch for MDEV-16629 while the original test case did fail. 2. In any case the test case should go to cte_recursive_not_embedded.test that was not created yet. --- mysql-test/r/cte_recursive.result | 21 --------------------- mysql-test/t/cte_recursive.test | 31 ------------------------------- 2 files changed, 52 deletions(-) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index d715fa20a6f..c892c76b33e 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -3300,24 +3300,3 @@ SELECT func(); func() 1 DROP FUNCTION func; -# -# MDEV-15151: function with recursive CTE using no base tables -# (duplicate of MDEV-16661) -# -connection default; -CREATE TABLE t1 (id int KEY); -INSERT INTO t1 VALUES (0), (1),(2); -CREATE OR REPLACE FUNCTION func() RETURNS int -RETURN -( -WITH recursive cte AS -(SELECT 1 a UNION SELECT cte.* FROM cte natural join t1) -SELECT * FROM cte limit 1 -); -connect con1,localhost,root,,; -SELECT func(); -connection default; -KILL QUERY 5; -DROP FUNCTION func; -DROP TABLE t1; -disconnect con1; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 227ccd0cdb3..4eee9ef2214 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1,4 +1,3 @@ ---source include/not_embedded.inc create table t1 (a int, b varchar(32)); insert into t1 values (4,'aaaa' ), (7,'bb'), (1,'ccc'), (4,'dd'); @@ -2324,33 +2323,3 @@ RETURN SELECT func(); DROP FUNCTION func; - ---echo # ---echo # MDEV-15151: function with recursive CTE using no base tables ---echo # (duplicate of MDEV-16661) ---echo # - ---connection default - -CREATE TABLE t1 (id int KEY); -INSERT INTO t1 VALUES (0), (1),(2); - -CREATE OR REPLACE FUNCTION func() RETURNS int -RETURN -( - WITH recursive cte AS - (SELECT 1 a UNION SELECT cte.* FROM cte natural join t1) - SELECT * FROM cte limit 1 -); - ---connect (con1,localhost,root,,) ---let $conid= `SELECT CONNECTION_ID()` ---send SELECT func() - ---connection default ---eval KILL QUERY $conid ---source include/restart_mysqld.inc - -DROP FUNCTION func; -DROP TABLE t1; ---disconnect con1 From e3207b6c132666e734adcd6e4c8485b1580039c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 6 Jul 2018 09:01:24 +0300 Subject: [PATCH 12/29] MDEV-14188 mariabackup.incremental_encrypted wrong result Add an explicit redo log flush. In this test innodb_flush_log_at_trx_commit was 2 by default. It is also possible that this failure occurs because of MDEV-15740. --- mysql-test/suite/mariabackup/incremental_encrypted.result | 1 + mysql-test/suite/mariabackup/incremental_encrypted.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.result b/mysql-test/suite/mariabackup/incremental_encrypted.result index e8f81e9fa49..e9525c9c4b7 100644 --- a/mysql-test/suite/mariabackup/incremental_encrypted.result +++ b/mysql-test/suite/mariabackup/incremental_encrypted.result @@ -2,6 +2,7 @@ call mtr.add_suppression("InnoDB: New log files created"); CREATE TABLE t(i INT) ENGINE INNODB ENCRYPTED=YES; INSERT INTO t VALUES(1); # Create full backup , modify table, then create incremental/differential backup +SET GLOBAL innodb_flush_log_at_trx_commit = 1; INSERT INTO t VALUES(2); SELECT * FROM t; i diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.test b/mysql-test/suite/mariabackup/incremental_encrypted.test index 8bcada493c6..e618ac4f79e 100644 --- a/mysql-test/suite/mariabackup/incremental_encrypted.test +++ b/mysql-test/suite/mariabackup/incremental_encrypted.test @@ -20,6 +20,7 @@ echo # Create full backup , modify table, then create incremental/differential b exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; --enable_result_log +SET GLOBAL innodb_flush_log_at_trx_commit = 1; INSERT INTO t VALUES(2); SELECT * FROM t; From 8b0d4cff0760b0a35285c315d82c49631c108baf Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 6 Jul 2018 17:13:53 +0300 Subject: [PATCH 13/29] MDEV-15855 Deadlock between purge thread and DDL statement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: ======== Truncate operation holds MDL on the table (t1) and tries to acquire InnoDB dict_operation_lock. Purge holds dict_operation_lock and tries to acquire MDL on the table (t1) to evaluate virtual column expressions for indexed virtual columns. It leads to deadlock of purge and truncate table (DDL). Solution: ========= If purge tries to acquire MDL on the table then it should do the following: i) Purge should release all innodb latches (including dict_operation_lock) before acquiring metadata lock on the table. ii) After acquiring metadata lock on the table, it should check whether the table was dropped or renamed. If the table is dropped then purge should ignore the undo log record. If the table is renamed then it should release the old MDL and acquire MDL on the new name. iii) Once purge acquires MDL, it should use the SQL table handle for all the remaining virtual index for the purge record. purge_node_t: Introduce new virtual column information to know whether the MDL was acquired successfully. This is joint work with Marko Mäkelä. --- .../gcol/r/innodb_virtual_debug_purge.result | 32 ++- .../gcol/t/innodb_virtual_debug_purge.test | 44 +++- sql/sql_class.cc | 7 - storage/innobase/handler/ha_innodb.cc | 209 +++++++++++++----- storage/innobase/include/btr0btr.h | 14 +- storage/innobase/include/btr0pcur.h | 15 ++ storage/innobase/include/btr0pcur.ic | 26 +++ storage/innobase/include/row0purge.h | 38 +++- storage/innobase/include/row0types.h | 52 +++++ storage/innobase/include/row0vers.h | 38 ++-- storage/innobase/row/row0purge.cc | 178 +++++++++++++-- storage/innobase/row/row0trunc.cc | 2 + storage/innobase/row/row0umod.cc | 2 +- storage/innobase/row/row0vers.cc | 146 ++++++++---- 14 files changed, 636 insertions(+), 167 deletions(-) diff --git a/mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result b/mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result index 309d8e8f04a..3fbc4576ece 100644 --- a/mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result +++ b/mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result @@ -175,8 +175,38 @@ SET DEBUG_SYNC='now WAIT_FOR halfway'; COMMIT; InnoDB 0 transactions not purged SET DEBUG_SYNC='now SIGNAL purged'; -disconnect prevent_purge; connection default; DROP TABLE t1; +CREATE TABLE t1 (y YEAR, vy YEAR AS (y) VIRTUAL UNIQUE, pk INT PRIMARY KEY) +ENGINE=InnoDB; +INSERT INTO t1 (pk,y) VALUES (1,2022); +CREATE TABLE t2(f1 INT NOT NULL, PRIMARY KEY(f1))ENGINE=InnoDB; +SET GLOBAL debug_dbug = '+d,ib_purge_virtual_index_callback'; +BEGIN; +INSERT INTO t2(f1) VALUES(1); +connection prevent_purge; +SET DEBUG_SYNC=RESET; +start transaction with consistent snapshot; +connection default; +COMMIT; +connect truncate,localhost,root,,; +REPLACE INTO t1(pk, y) SELECT pk,y FROM t1; +SET DEBUG_SYNC='row_trunc_before_dict_lock SIGNAL commit WAIT_FOR release'; +TRUNCATE TABLE t1; +connection prevent_purge; +SET DEBUG_SYNC='now WAIT_FOR commit'; +COMMIT; +SET DEBUG_SYNC='now SIGNAL purge_start'; +disconnect prevent_purge; +connection default; +SET DEBUG_SYNC='now WAIT_FOR purge_start'; +InnoDB 1 transactions not purged +SET DEBUG_SYNC='now SIGNAL release'; +SET GLOBAL debug_dbug=@old_dbug; +connection truncate; +disconnect truncate; +connection default; +InnoDB 0 transactions not purged +DROP TABLE t1, t2; set debug_sync=reset; SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency; diff --git a/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test b/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test index 7feeee768ff..845881a6d07 100644 --- a/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test +++ b/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test @@ -217,12 +217,54 @@ SET DEBUG_SYNC='now WAIT_FOR halfway'; COMMIT; --source ../../innodb/include/wait_all_purged.inc SET DEBUG_SYNC='now SIGNAL purged'; -disconnect prevent_purge; connection default; reap; DROP TABLE t1; +CREATE TABLE t1 (y YEAR, vy YEAR AS (y) VIRTUAL UNIQUE, pk INT PRIMARY KEY) +ENGINE=InnoDB; + +INSERT INTO t1 (pk,y) VALUES (1,2022); +CREATE TABLE t2(f1 INT NOT NULL, PRIMARY KEY(f1))ENGINE=InnoDB; + +SET GLOBAL debug_dbug = '+d,ib_purge_virtual_index_callback'; + +BEGIN; +INSERT INTO t2(f1) VALUES(1); +connection prevent_purge; +SET DEBUG_SYNC=RESET; +start transaction with consistent snapshot; +connection default; +COMMIT; + +connect(truncate,localhost,root,,); +REPLACE INTO t1(pk, y) SELECT pk,y FROM t1; +SET DEBUG_SYNC='row_trunc_before_dict_lock SIGNAL commit WAIT_FOR release'; +send TRUNCATE TABLE t1; + +connection prevent_purge; +SET DEBUG_SYNC='now WAIT_FOR commit'; +COMMIT; +SET DEBUG_SYNC='now SIGNAL purge_start'; +disconnect prevent_purge; + +connection default; +SET DEBUG_SYNC='now WAIT_FOR purge_start'; +let $wait_all_purged=1; +--source ../../innodb/include/wait_all_purged.inc +let $wait_all_purged=0; +SET DEBUG_SYNC='now SIGNAL release'; +SET GLOBAL debug_dbug=@old_dbug; + +connection truncate; +reap; +disconnect truncate; + +connection default; +--source ../../innodb/include/wait_all_purged.inc +DROP TABLE t1, t2; + --source include/wait_until_count_sessions.inc set debug_sync=reset; SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index a1010fb4c35..bd41a3729d6 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4404,13 +4404,6 @@ TABLE *open_purge_table(THD *thd, const char *db, size_t dblen, DBUG_RETURN(error ? NULL : tl->table); } -TABLE *get_purge_table(THD *thd) -{ - /* see above, at most one table can be opened */ - DBUG_ASSERT(thd->open_tables == NULL || thd->open_tables->next == NULL); - return thd->open_tables; -} - /** Find an open table in the list of prelocked tabled diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 7cba4ef5794..e9d7a6d94df 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -129,7 +129,7 @@ void destroy_thd(MYSQL_THD thd); void reset_thd(MYSQL_THD thd); TABLE *open_purge_table(THD *thd, const char *db, size_t dblen, const char *tb, size_t tblen); -TABLE *get_purge_table(THD *thd); +void close_thread_tables(THD* thd); /** Check if user has used xtradb extended system variable that is not currently supported by innodb or marked as deprecated. */ @@ -21603,63 +21603,154 @@ innobase_index_cond( return handler_index_cond_check(file); } - -/** Find or open a mysql table for the virtual column template -@param[in] thd mysql thread handle -@param[in,out] table InnoDB table whose virtual column template is to be updated -@return TABLE if successful or NULL */ -static TABLE * -innobase_find_mysql_table_for_vc( -/*=============================*/ - THD* thd, - dict_table_t* table) +/** Parse the table file name into table name and database name. +@param[in] tbl_name InnoDB table name +@param[out] dbname database name buffer (NAME_LEN + 1 bytes) +@param[out] tblname table name buffer (NAME_LEN + 1 bytes) +@param[out] dbnamelen database name length +@param[out] tblnamelen table name length +@return true if the table name is parsed properly. */ +static bool table_name_parse( + const table_name_t& tbl_name, + char* dbname, + char* tblname, + ulint& dbnamelen, + ulint& tblnamelen) { - TABLE *mysql_table; - bool bg_thread = THDVAR(thd, background_thread); + dbnamelen = dict_get_db_name_len(tbl_name.m_name); + char db_buf[MAX_DATABASE_NAME_LEN + 1]; + char tbl_buf[MAX_TABLE_NAME_LEN + 1]; - if (bg_thread) { - if ((mysql_table = get_purge_table(thd))) { - return mysql_table; + ut_ad(dbnamelen > 0); + ut_ad(dbnamelen <= MAX_DATABASE_NAME_LEN); + + memcpy(db_buf, tbl_name.m_name, dbnamelen); + db_buf[dbnamelen] = 0; + + tblnamelen = strlen(tbl_name.m_name + dbnamelen + 1); + memcpy(tbl_buf, tbl_name.m_name + dbnamelen + 1, tblnamelen); + tbl_buf[tblnamelen] = 0; + + filename_to_tablename(db_buf, dbname, MAX_DATABASE_NAME_LEN + 1, true); + + if (tblnamelen > TEMP_FILE_PREFIX_LENGTH + && !strncmp(tbl_buf, TEMP_FILE_PREFIX, TEMP_FILE_PREFIX_LENGTH)) { + return false; + } + + if (char *is_part = strchr(tbl_buf, '#')) { + *is_part = '\0'; + } + + filename_to_tablename(tbl_buf, tblname, MAX_TABLE_NAME_LEN + 1, true); + return true; +} + + +/** Acquire metadata lock and MariaDB table handle for an InnoDB table. +@param[in,out] thd thread handle +@param[in,out] table InnoDB table +@return MariaDB table handle +@retval NULL if the table does not exist, is unaccessible or corrupted. */ +static TABLE* innodb_acquire_mdl(THD* thd, dict_table_t* table) +{ + char db_buf[NAME_LEN + 1], db_buf1[NAME_LEN + 1]; + char tbl_buf[NAME_LEN + 1], tbl_buf1[NAME_LEN + 1]; + ulint db_buf_len, db_buf1_len; + ulint tbl_buf_len, tbl_buf1_len; + + if (!table_name_parse(table->name, db_buf, tbl_buf, + db_buf_len, tbl_buf_len)) { + ut_ad(!"invalid table name"); + return NULL; + } + + const table_id_t table_id = table->id; +retry_mdl: + const bool unaccessible = !table->is_readable() || table->corrupted; + table->release(); + + if (unaccessible) { + return NULL; + } + + TABLE* mariadb_table = open_purge_table(thd, db_buf, db_buf_len, + tbl_buf, tbl_buf_len); + + table = dict_table_open_on_id(table_id, false, DICT_TABLE_OP_NORMAL); + + if (table == NULL) { + /* Table is dropped. */ + goto fail; + } + + if (!fil_table_accessible(table)) { +release_fail: + table->release(); +fail: + if (mariadb_table) { + close_thread_tables(thd); } + + return NULL; + } + + if (!table_name_parse(table->name, db_buf1, tbl_buf1, + db_buf1_len, tbl_buf1_len)) { + ut_ad(!"invalid table name"); + goto release_fail; + } + + if (!mariadb_table) { + } else if (!strcmp(db_buf, db_buf1) && !strcmp(tbl_buf, tbl_buf1)) { + return mariadb_table; } else { - if (table->vc_templ->mysql_table_query_id == thd_get_query_id(thd)) { + /* Table is renamed. So release MDL for old name and try + to acquire the MDL for new table name. */ + close_thread_tables(thd); + } + + strcpy(tbl_buf, tbl_buf1); + strcpy(db_buf, db_buf1); + tbl_buf_len = tbl_buf1_len; + db_buf_len = db_buf1_len; + goto retry_mdl; +} + +/** Find or open a table handle for the virtual column template +@param[in] thd thread handle +@param[in,out] table InnoDB table whose virtual column template + is to be updated +@return table handle +@retval NULL if the table is dropped, unaccessible or corrupted +for purge thread */ +static TABLE* innodb_find_table_for_vc(THD* thd, dict_table_t* table) +{ + if (THDVAR(thd, background_thread)) { + /* Purge thread acquires dict_operation_lock while + processing undo log record. Release the dict_operation_lock + before acquiring MDL on the table. */ + rw_lock_s_unlock(dict_operation_lock); + return innodb_acquire_mdl(thd, table); + } else { + if (table->vc_templ->mysql_table_query_id + == thd_get_query_id(thd)) { return table->vc_templ->mysql_table; } } - char dbname[MAX_DATABASE_NAME_LEN + 1]; - char tbname[MAX_TABLE_NAME_LEN + 1]; - char* name = table->name.m_name; - uint dbnamelen = (uint) dict_get_db_name_len(name); - uint tbnamelen = (uint) strlen(name) - dbnamelen - 1; - char t_dbname[MAX_DATABASE_NAME_LEN + 1]; - char t_tbname[MAX_TABLE_NAME_LEN + 1]; + char db_buf[NAME_LEN + 1]; + char tbl_buf[NAME_LEN + 1]; + ulint db_buf_len, tbl_buf_len; - strncpy(dbname, name, dbnamelen); - dbname[dbnamelen] = 0; - strncpy(tbname, name + dbnamelen + 1, tbnamelen); - tbname[tbnamelen] =0; - - /* For partition table, remove the partition name and use the - "main" table name to build the template */ - char* is_part = is_partition(tbname); - - if (is_part != NULL) { - *is_part = '\0'; + if (!table_name_parse(table->name, db_buf, tbl_buf, + db_buf_len, tbl_buf_len)) { + ut_ad(!"invalid table name"); + return NULL; } - dbnamelen = filename_to_tablename(dbname, t_dbname, - MAX_DATABASE_NAME_LEN + 1); - tbnamelen = filename_to_tablename(tbname, t_tbname, - MAX_TABLE_NAME_LEN + 1); - - if (bg_thread) { - return open_purge_table(thd, t_dbname, dbnamelen, - t_tbname, tbnamelen); - } - - mysql_table = find_fk_open_table(thd, t_dbname, dbnamelen, - t_tbname, tbnamelen); + TABLE* mysql_table = find_fk_open_table(thd, db_buf, db_buf_len, + tbl_buf, tbl_buf_len); table->vc_templ->mysql_table = mysql_table; table->vc_templ->mysql_table_query_id = thd_get_query_id(thd); @@ -21678,7 +21769,7 @@ innobase_init_vc_templ( table->vc_templ = UT_NEW_NOKEY(dict_vcol_templ_t()); - TABLE *mysql_table= innobase_find_mysql_table_for_vc(current_thd, table); + TABLE *mysql_table= innodb_find_table_for_vc(current_thd, table); ut_ad(mysql_table); if (!mysql_table) { @@ -21772,15 +21863,16 @@ innobase_get_field_from_update_vector( Allocate a heap and record for calculating virtual fields Used mainly for virtual fields in indexes -@param[in] thd MariaDB THD -@param[in] index Index in use +@param[in] thd MariaDB THD +@param[in] index Index in use @param[out] heap Heap that holds temporary row -@param[in,out] mysql_table MariaDB table -@param[out] rec Pointer to allocated MariaDB record -@param[out] storage Internal storage for blobs etc +@param[in,out] table MariaDB table +@param[out] record Pointer to allocated MariaDB record +@param[out] storage Internal storage for blobs etc -@return FALSE ok -@return TRUE malloc failure +@retval false on success +@retval true on malloc failure or failed to open the maria table + for purge thread. */ bool innobase_allocate_row_for_vcol( @@ -21794,7 +21886,12 @@ bool innobase_allocate_row_for_vcol( TABLE *maria_table; String *blob_value_storage; if (!*table) - *table= innobase_find_mysql_table_for_vc(thd, index->table); + *table= innodb_find_table_for_vc(thd, index->table); + + /* For purge thread, there is a possiblity that table could have + dropped, corrupted or unaccessible. */ + if (!*table) + return true; maria_table= *table; if (!*heap && !(*heap= mem_heap_create(srv_page_size))) { diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index 1d7710a1496..2fccdfc431c 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -115,7 +115,15 @@ enum btr_latch_mode { /** Attempt to purge a secondary index record while holding the dict_index_t::lock S-latch. */ BTR_PURGE_LEAF_ALREADY_S_LATCHED = BTR_PURGE_LEAF - | BTR_ALREADY_S_LATCHED + | BTR_ALREADY_S_LATCHED, + + /** In the case of BTR_MODIFY_TREE, the caller specifies + the intention to delete record only. It is used to optimize + block->lock range.*/ + BTR_LATCH_FOR_DELETE = 65536, + + /** Attempt to purge a secondary index record in the tree. */ + BTR_PURGE_TREE = BTR_MODIFY_TREE | BTR_LATCH_FOR_DELETE }; /** This flag ORed to btr_latch_mode says that we do the search in query @@ -131,10 +139,6 @@ the insert buffer to speed up inserts */ to insert record only. It is used to optimize block->lock range.*/ #define BTR_LATCH_FOR_INSERT 32768U -/** In the case of BTR_MODIFY_TREE, the caller specifies the intention -to delete record only. It is used to optimize block->lock range.*/ -#define BTR_LATCH_FOR_DELETE 65536U - /** This flag is for undo insert of rtree. For rtree, we need this flag to find proper rec to undo insert.*/ #define BTR_RTREE_UNDO_INS 131072U diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index fab934ca0ee..f9f132b1733 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -296,6 +296,21 @@ btr_pcur_commit_specify_mtr( /*========================*/ btr_pcur_t* pcur, /*!< in: persistent cursor */ mtr_t* mtr); /*!< in: mtr to commit */ + +/** Commits the mtr and sets the clustered index pcur and secondary index +pcur latch mode to BTR_NO_LATCHES, that is, the cursor becomes detached. +Function btr_pcur_store_position should be used for both cursor before +calling this, if restoration of cursor is wanted later. +@param[in] pcur persistent cursor +@param[in] sec_pcur secondary index persistent cursor +@param[in] mtr mtr to commit */ +UNIV_INLINE +void +btr_pcurs_commit_specify_mtr( + btr_pcur_t* pcur, + btr_pcur_t* sec_pcur, + mtr_t* mtr); + /*********************************************************//** Moves the persistent cursor to the next record in the tree. If no records are left, the cursor stays 'after last in tree'. diff --git a/storage/innobase/include/btr0pcur.ic b/storage/innobase/include/btr0pcur.ic index 4490942a2bb..b2a85def63d 100644 --- a/storage/innobase/include/btr0pcur.ic +++ b/storage/innobase/include/btr0pcur.ic @@ -389,6 +389,32 @@ btr_pcur_commit_specify_mtr( pcur->pos_state = BTR_PCUR_WAS_POSITIONED; } +/** Commits the mtr and sets the clustered index pcur and secondary index +pcur latch mode to BTR_NO_LATCHES, that is, the cursor becomes detached. +Function btr_pcur_store_position should be used for both cursor before +calling this, if restoration of cursor is wanted later. +@param[in] pcur persistent cursor +@param[in] sec_pcur secondary index persistent cursor +@param[in] mtr mtr to commit */ +UNIV_INLINE +void +btr_pcurs_commit_specify_mtr( + btr_pcur_t* pcur, + btr_pcur_t* sec_pcur, + mtr_t* mtr) +{ + ut_ad(pcur->pos_state == BTR_PCUR_IS_POSITIONED); + ut_ad(sec_pcur->pos_state == BTR_PCUR_IS_POSITIONED); + + pcur->latch_mode = BTR_NO_LATCHES; + sec_pcur->latch_mode = BTR_NO_LATCHES; + + mtr_commit(mtr); + + pcur->pos_state = BTR_PCUR_WAS_POSITIONED; + sec_pcur->pos_state = BTR_PCUR_WAS_POSITIONED; +} + /**************************************************************//** Sets the old_rec_buf field to NULL. */ UNIV_INLINE diff --git a/storage/innobase/include/row0purge.h b/storage/innobase/include/row0purge.h index 3cf4a7b982a..655685c02a8 100644 --- a/storage/innobase/include/row0purge.h +++ b/storage/innobase/include/row0purge.h @@ -36,6 +36,7 @@ Created 3/14/1997 Heikki Tuuri #include "que0types.h" #include "row0types.h" #include "ut0vec.h" +#include "row0mysql.h" /** Create a purge node to a query graph. @param[in] parent parent node, i.e., a thr node @@ -47,8 +48,7 @@ row_purge_node_create( mem_heap_t* heap) MY_ATTRIBUTE((warn_unused_result)); -/***********************************************************//** -Determines if it is possible to remove a secondary index entry. +/** Determines if it is possible to remove a secondary index entry. Removal is possible if the secondary index entry does not refer to any not delete marked version of a clustered index record where DB_TRX_ID is newer than the purge view. @@ -61,14 +61,27 @@ inserts a record that the secondary index entry would refer to. However, in that case, the user transaction would also re-insert the secondary index entry after purge has removed it and released the leaf page latch. +@param[in,out] node row purge node +@param[in] index secondary index +@param[in] entry secondary index entry +@param[in,out] sec_pcur secondary index cursor or NULL + if it is called for purge buffering + operation. +@param[in,out] sec_mtr mini-transaction which holds + secondary index entry or NULL if it is + called for purge buffering operation. +@param[in] is_tree true=pessimistic purge, + false=optimistic (leaf-page only) @return true if the secondary index record can be purged */ bool row_purge_poss_sec( -/*===============*/ - purge_node_t* node, /*!< in/out: row purge node */ - dict_index_t* index, /*!< in: secondary index */ - const dtuple_t* entry) /*!< in: secondary index entry */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + purge_node_t* node, + dict_index_t* index, + const dtuple_t* entry, + btr_pcur_t* sec_pcur=NULL, + mtr_t* sec_mtr=NULL, + bool is_tree=false); + /*************************************************************** Does the purge operation for a single undo log record. This is a high-level function used in an SQL execution graph. @@ -117,6 +130,10 @@ struct purge_node_t{ ibool done; /* Debug flag */ trx_id_t trx_id; /*!< trx id for this purging record */ + /** Virtual column information about opening of MariaDB table. + It resets after processing each undo log record. */ + purge_vcol_info_t vcol_info; + #ifdef UNIV_DEBUG /***********************************************************//** Validate the persisent cursor. The purge node has two references @@ -125,8 +142,13 @@ struct purge_node_t{ each other if the found_clust flag is set. @return true if the persistent cursor is consistent with the ref member.*/ - bool validate_pcur(); + bool validate_pcur(); #endif + + /** Whether purge failed to open the maria table for virtual column + computation. + @return true if the table failed to open. */ + bool vcol_op_failed() const { return !vcol_info.validate(); } }; #endif diff --git a/storage/innobase/include/row0types.h b/storage/innobase/include/row0types.h index 52c89cb01fa..1bff3767253 100644 --- a/storage/innobase/include/row0types.h +++ b/storage/innobase/include/row0types.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -52,4 +53,55 @@ struct row_log_t; /* MySQL data types */ struct TABLE; +/** Purge virtual column node information. */ +struct purge_vcol_info_t +{ + /** Is there a possible need to evaluate virtual columns? */ + bool requested; + /** Do we have to evaluate virtual columns (using mariadb_table)? */ + bool used; + + /** True if it is used for the first time. */ + bool first_use; + + /** MariaDB table opened for virtual column computation. */ + TABLE* mariadb_table; + + /** Reset the state. */ + void reset() + { + requested = false; + used = false; + first_use = false; + mariadb_table = NULL; + } + + /** Validate the virtual column information. + @return true if the mariadb table opened successfully + or doesn't try to calculate virtual column. */ + bool validate() const { return !used || mariadb_table; } + + /** Note that the virtual column information is needed. */ + void set_used() + { + ut_ad(requested); + + if (first_use) { + first_use = false; + ut_ad(used); + return; + } + + first_use = used = true; + } + + /** Check whether it fetches mariadb table for the first time. + @return true if first time tries to open mariadb table. */ + bool is_first_fetch() const + { + ut_ad(!first_use || used); + return first_use; + } +}; + #endif diff --git a/storage/innobase/include/row0vers.h b/storage/innobase/include/row0vers.h index 576b53358f8..23c2e8546bc 100644 --- a/storage/innobase/include/row0vers.h +++ b/storage/innobase/include/row0vers.h @@ -35,6 +35,7 @@ Created 2/6/1997 Heikki Tuuri #include "rem0types.h" #include "mtr0mtr.h" #include "dict0mem.h" +#include "row0types.h" // Forward declaration class ReadView; @@ -68,27 +69,34 @@ row_vers_must_preserve_del_marked( const table_name_t& name, mtr_t* mtr); -/*****************************************************************//** -Finds out if a version of the record, where the version >= the current +/** Finds out if a version of the record, where the version >= the current purge view, should have ientry as its secondary index entry. We check if there is any not delete marked version of the record where the trx id >= purge view, and the secondary index entry == ientry; exactly in this case we return TRUE. +@param[in] also_curr TRUE if also rec is included in the versions + to search; otherwise only versions prior + to it are searched +@param[in] rec record in the clustered index; the caller + must have a latch on the page +@param[in] mtr mtr holding the latch on rec; it will + also hold the latch on purge_view +@param[in] index secondary index +@param[in] ientry secondary index entry +@param[in] roll_ptr roll_ptr for the purge record +@param[in] trx_id transaction ID on the purging record +@param[in,out] vcol_info virtual column information for purge thread. @return TRUE if earlier version should have */ -ibool +bool row_vers_old_has_index_entry( -/*=========================*/ - ibool also_curr,/*!< in: TRUE if also rec is included in the - versions to search; otherwise only versions - prior to it are searched */ - const rec_t* rec, /*!< in: record in the clustered index; the - caller must have a latch on the page */ - mtr_t* mtr, /*!< in: mtr holding the latch on rec; it will - also hold the latch on purge_view */ - dict_index_t* index, /*!< in: the secondary index */ - const dtuple_t* ientry, /*!< in: the secondary index entry */ - roll_ptr_t roll_ptr,/*!< in: roll_ptr for the purge record */ - trx_id_t trx_id);/*!< in: transaction ID on the purging record */ + bool also_curr, + const rec_t* rec, + mtr_t* mtr, + dict_index_t* index, + const dtuple_t* ientry, + roll_ptr_t roll_ptr, + trx_id_t trx_id, + purge_vcol_info_t* vcol_info=NULL); /*****************************************************************//** Constructs the version of a clustered index record which a consistent diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 448e2984b41..8fe0a59eb3c 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -136,7 +136,8 @@ row_purge_remove_clust_if_poss_low( ulint offsets_[REC_OFFS_NORMAL_SIZE]; rec_offs_init(offsets_); - ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_S)); + ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_S) + || node->vcol_info.used); index = dict_table_get_first_index(node->table); @@ -230,8 +231,55 @@ row_purge_remove_clust_if_poss( return(false); } -/***********************************************************//** -Determines if it is possible to remove a secondary index entry. +/** Tries to store secondary index cursor before openin mysql table for +virtual index condition computation. +@param[in,out] node row purge node +@param[in] index secondary index +@param[in,out] sec_pcur secondary index cursor +@param[in,out] sec_mtr mini-transaction which holds + secondary index entry */ +static void row_purge_store_vsec_cur( + purge_node_t* node, + dict_index_t* index, + btr_pcur_t* sec_pcur, + mtr_t* sec_mtr) +{ + row_purge_reposition_pcur(BTR_SEARCH_LEAF, node, sec_mtr); + + if (!node->found_clust) { + return; + } + + node->vcol_info.requested = true; + + btr_pcur_store_position(sec_pcur, sec_mtr); + + btr_pcurs_commit_specify_mtr(&node->pcur, sec_pcur, sec_mtr); +} + +/** Tries to restore secondary index cursor after opening the mysql table +@param[in,out] node row purge node +@param[in] index secondary index +@param[in,out] sec_mtr mini-transaction which holds secondary index entry +@param[in] is_tree true=pessimistic purge, + false=optimistic (leaf-page only) +@return false in case of restore failure. */ +static bool row_purge_restore_vsec_cur( + purge_node_t* node, + dict_index_t* index, + btr_pcur_t* sec_pcur, + mtr_t* sec_mtr, + bool is_tree) +{ + sec_mtr->start(); + sec_mtr->set_named_space(index->space); + + return btr_pcur_restore_position( + is_tree ? BTR_PURGE_TREE : BTR_PURGE_LEAF, + sec_pcur, sec_mtr); +} + +/** Determines if it is possible to remove a secondary index entry. Removal is possible if the secondary index entry does not refer to any not delete marked version of a clustered index record where DB_TRX_ID is newer than the purge view. @@ -244,25 +292,66 @@ inserts a record that the secondary index entry would refer to. However, in that case, the user transaction would also re-insert the secondary index entry after purge has removed it and released the leaf page latch. +@param[in,out] node row purge node +@param[in] index secondary index +@param[in] entry secondary index entry +@param[in,out] sec_pcur secondary index cursor or NULL + if it is called for purge buffering + operation. +@param[in,out] sec_mtr mini-transaction which holds + secondary index entry or NULL if it is + called for purge buffering operation. +@param[in] is_tree true=pessimistic purge, + false=optimistic (leaf-page only) @return true if the secondary index record can be purged */ bool row_purge_poss_sec( -/*===============*/ - purge_node_t* node, /*!< in/out: row purge node */ - dict_index_t* index, /*!< in: secondary index */ - const dtuple_t* entry) /*!< in: secondary index entry */ + purge_node_t* node, + dict_index_t* index, + const dtuple_t* entry, + btr_pcur_t* sec_pcur, + mtr_t* sec_mtr, + bool is_tree) { bool can_delete; mtr_t mtr; ut_ad(!dict_index_is_clust(index)); + + const bool store_cur = sec_mtr && !node->vcol_info.used + && dict_index_has_virtual(index); + + if (store_cur) { + row_purge_store_vsec_cur(node, index, sec_pcur, sec_mtr); + + /* The PRIMARY KEY value was not found in the clustered + index. The secondary index record found. We can purge + the secondary index record. */ + if (!node->vcol_info.requested) { + ut_ad(!node->found_clust); + return true; + } + } + +retry_purge_sec: mtr_start(&mtr); can_delete = !row_purge_reposition_pcur(BTR_SEARCH_LEAF, node, &mtr) - || !row_vers_old_has_index_entry(TRUE, + || !row_vers_old_has_index_entry(true, btr_pcur_get_rec(&node->pcur), &mtr, index, entry, - node->roll_ptr, node->trx_id); + node->roll_ptr, node->trx_id, + &node->vcol_info); + + if (node->vcol_info.is_first_fetch()) { + if (node->vcol_info.mariadb_table) { + goto retry_purge_sec; + } + + node->table = NULL; + sec_pcur = NULL; + return false; + } /* Persistent cursor is closed if reposition fails. */ if (node->found_clust) { @@ -271,7 +360,12 @@ row_purge_poss_sec( mtr_commit(&mtr); } - return(can_delete); + if (store_cur && !row_purge_restore_vsec_cur( + node, index, sec_pcur, sec_mtr, is_tree)) { + return false; + } + + return can_delete; } /*************************************************************** @@ -287,7 +381,6 @@ row_purge_remove_sec_if_poss_tree( const dtuple_t* entry) /*!< in: index entry */ { btr_pcur_t pcur; - btr_cur_t* btr_cur; ibool success = TRUE; dberr_t err; mtr_t mtr; @@ -348,16 +441,16 @@ row_purge_remove_sec_if_poss_tree( ut_error; } - btr_cur = btr_pcur_get_btr_cur(&pcur); - /* We should remove the index record if no later version of the row, which cannot be purged yet, requires its existence. If some requires, we should do nothing. */ - if (row_purge_poss_sec(node, index, entry)) { + if (row_purge_poss_sec(node, index, entry, &pcur, &mtr, true)) { + /* Remove the index record, which should have been marked for deletion. */ - if (!rec_get_deleted_flag(btr_cur_get_rec(btr_cur), + if (!rec_get_deleted_flag(btr_cur_get_rec( + btr_pcur_get_btr_cur(&pcur)), dict_table_is_comp(index->table))) { ib::error() << "tried to purge non-delete-marked record" @@ -365,15 +458,18 @@ row_purge_remove_sec_if_poss_tree( << " of table " << index->table->name << ": tuple: " << *entry << ", record: " << rec_index_print( - btr_cur_get_rec(btr_cur), index); + btr_cur_get_rec( + btr_pcur_get_btr_cur(&pcur)), + index); ut_ad(0); goto func_exit; } - btr_cur_pessimistic_delete(&err, FALSE, btr_cur, 0, - false, &mtr); + btr_cur_pessimistic_delete(&err, FALSE, + btr_pcur_get_btr_cur(&pcur), + 0, false, &mtr); switch (UNIV_EXPECT(err, DB_SUCCESS)) { case DB_SUCCESS: break; @@ -385,6 +481,13 @@ row_purge_remove_sec_if_poss_tree( } } + if (node->vcol_op_failed()) { + ut_ad(mtr.has_committed()); + ut_ad(!pcur.old_rec_buf); + ut_ad(pcur.pos_state == BTR_PCUR_NOT_POSITIONED); + return false; + } + func_exit: btr_pcur_close(&pcur); func_exit_no_pcur: @@ -445,8 +548,10 @@ row_purge_remove_sec_if_poss_leaf( index->is_committed(). */ ut_ad(!dict_index_is_online_ddl(index)); - /* Change buffering is disabled for spatial index. */ - mode = dict_index_is_spatial(index) + /* Change buffering is disabled for spatial index and + virtual index. */ + mode = (dict_index_is_spatial(index) + || dict_index_has_virtual(index)) ? BTR_MODIFY_LEAF : BTR_PURGE_LEAF; } @@ -474,7 +579,7 @@ row_purge_remove_sec_if_poss_leaf( case ROW_FOUND: /* Before attempting to purge a record, check if it is safe to do so. */ - if (row_purge_poss_sec(node, index, entry)) { + if (row_purge_poss_sec(node, index, entry, &pcur, &mtr, false)) { btr_cur_t* btr_cur = btr_pcur_get_btr_cur(&pcur); /* Only delete-marked records should be purged. */ @@ -540,6 +645,12 @@ row_purge_remove_sec_if_poss_leaf( success = false; } } + + if (node->vcol_op_failed()) { + btr_pcur_close(&pcur); + return false; + } + /* (The index entry is still needed, or the deletion succeeded) */ /* fall through */ @@ -586,6 +697,10 @@ row_purge_remove_sec_if_poss( return; } retry: + if (node->vcol_op_failed()) { + return; + } + success = row_purge_remove_sec_if_poss_tree(node, index, entry); /* The delete operation may fail if we have little file space left: TODO: easiest to crash the database @@ -652,6 +767,12 @@ row_purge_del_mark( node->row, NULL, node->index, heap, ROW_BUILD_FOR_PURGE); row_purge_remove_sec_if_poss(node, node->index, entry); + + if (node->vcol_op_failed()) { + mem_heap_free(heap); + return false; + } + mem_heap_empty(heap); } @@ -678,7 +799,8 @@ row_purge_upd_exist_or_extern_func( { mem_heap_t* heap; - ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_S)); + ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_S) + || node->vcol_info.used); ut_ad(!node->table->skip_alter_undo); if (node->rec_type == TRX_UNDO_UPD_DEL_REC @@ -1016,10 +1138,15 @@ row_purge( bool purged = row_purge_record( node, undo_rec, thr, updated_extern); - rw_lock_s_unlock(dict_operation_lock); + if (!node->vcol_info.used) { + rw_lock_s_unlock(dict_operation_lock); + } + + ut_ad(!rw_lock_own(dict_operation_lock, RW_LOCK_S)); if (purged - || srv_shutdown_state != SRV_SHUTDOWN_NONE) { + || srv_shutdown_state != SRV_SHUTDOWN_NONE + || node->vcol_op_failed()) { return; } @@ -1051,6 +1178,8 @@ row_purge_end( node->done = TRUE; + node->vcol_info.reset(); + ut_a(thr->run_node != NULL); mem_heap_empty(node->heap); @@ -1098,6 +1227,7 @@ row_purge_step( row_purge_end(thr); } else { thr->run_node = node; + node->vcol_info.reset(); } } else { row_purge_end(thr); diff --git a/storage/innobase/row/row0trunc.cc b/storage/innobase/row/row0trunc.cc index 94be5152596..f6be8bdfbc5 100644 --- a/storage/innobase/row/row0trunc.cc +++ b/storage/innobase/row/row0trunc.cc @@ -1810,6 +1810,8 @@ row_truncate_table_for_mysql( trx_start_for_ddl(trx, TRX_DICT_OP_TABLE); } + DEBUG_SYNC_C("row_trunc_before_dict_lock"); + /* Step-3: Validate ownership of needed locks (Exclusive lock). Ownership will also ensure there is no active SQL queries, INSERT, SELECT, .....*/ diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index 6fc72f806e0..5f28f096286 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -508,7 +508,7 @@ row_undo_mod_del_mark_or_remove_sec_low( any no-redo rollback segment undo logs. */ if (dict_table_is_temporary(node->table) || row_vers_old_has_index_entry( - FALSE, btr_pcur_get_rec(&(node->pcur)), + false, btr_pcur_get_rec(&(node->pcur)), &mtr_vers, index, entry, 0, 0)) { err = btr_cur_del_mark_set_sec_rec(BTR_NO_LOCKING_FLAG, btr_cur, TRUE, thr, &mtr); diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index aebcf70715f..9ced22fd54b 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -436,14 +436,16 @@ row_vers_must_preserve_del_marked( @param[in,out] row the cluster index row in dtuple form @param[in] clust_index clustered index @param[in] index the secondary index -@param[in] heap heap used to build virtual dtuple */ +@param[in] heap heap used to build virtual dtuple +@param[in,out] vcol_info virtual column information. */ static void row_vers_build_clust_v_col( - dtuple_t* row, - dict_index_t* clust_index, - dict_index_t* index, - mem_heap_t* heap) + dtuple_t* row, + dict_index_t* clust_index, + dict_index_t* index, + mem_heap_t* heap, + purge_vcol_info_t* vcol_info) { mem_heap_t* local_heap = NULL; VCOL_STORAGE *vcol_storage= NULL; @@ -453,12 +455,23 @@ row_vers_build_clust_v_col( ut_ad(dict_index_has_virtual(index)); + if (vcol_info != NULL) { + vcol_info->set_used(); + maria_table = vcol_info->mariadb_table; + } + innobase_allocate_row_for_vcol(thd, index, &local_heap, &maria_table, &record, &vcol_storage); + if (vcol_info && !vcol_info->mariadb_table) { + vcol_info->mariadb_table = maria_table; + ut_ad(!maria_table || vcol_info->is_first_fetch()); + goto func_exit; + } + for (ulint i = 0; i < dict_index_get_n_fields(index); i++) { const dict_field_t* ind_field = dict_index_get_nth_field( index, i); @@ -476,6 +489,7 @@ row_vers_build_clust_v_col( } } +func_exit: if (local_heap) { if (vcol_storage) innobase_free_row_for_vcol(vcol_storage); @@ -497,16 +511,16 @@ row_vers_build_clust_v_col( static void row_vers_build_cur_vrow_low( - bool in_purge, - const rec_t* rec, - dict_index_t* clust_index, - ulint* clust_offsets, - dict_index_t* index, - roll_ptr_t roll_ptr, - trx_id_t trx_id, - mem_heap_t* v_heap, - const dtuple_t**vrow, - mtr_t* mtr) + bool in_purge, + const rec_t* rec, + dict_index_t* clust_index, + ulint* clust_offsets, + dict_index_t* index, + roll_ptr_t roll_ptr, + trx_id_t trx_id, + mem_heap_t* v_heap, + const dtuple_t** vrow, + mtr_t* mtr) { const rec_t* version; rec_t* prev_version; @@ -784,21 +798,23 @@ func_exit: @param[in,out] heap heap memory @param[in,out] v_heap heap memory to keep virtual colum dtuple @param[in] mtr mtr holding the latch on rec +@param[in,out] vcol_info virtual column information for purge thread @return dtuple contains virtual column data */ static const dtuple_t* row_vers_build_cur_vrow( - bool in_purge, - const rec_t* rec, - dict_index_t* clust_index, - ulint** clust_offsets, - dict_index_t* index, - const dtuple_t* ientry, - roll_ptr_t roll_ptr, - trx_id_t trx_id, - mem_heap_t* heap, - mem_heap_t* v_heap, - mtr_t* mtr) + bool in_purge, + const rec_t* rec, + dict_index_t* clust_index, + ulint** clust_offsets, + dict_index_t* index, + const dtuple_t* ientry, + roll_ptr_t roll_ptr, + trx_id_t trx_id, + mem_heap_t* heap, + mem_heap_t* v_heap, + mtr_t* mtr, + purge_vcol_info_t* vcol_info) { const dtuple_t* cur_vrow = NULL; @@ -818,8 +834,17 @@ row_vers_build_cur_vrow( rec, *clust_offsets, NULL, NULL, NULL, NULL, heap); + if (vcol_info && !vcol_info->used) { + mtr->commit(); + } + row_vers_build_clust_v_col( - row, clust_index, index, heap); + row, clust_index, index, heap, vcol_info); + + if (vcol_info != NULL && vcol_info->is_first_fetch()) { + return NULL; + } + cur_vrow = dtuple_copy(row, v_heap); dtuple_dup_v_fld(cur_vrow, v_heap); } else { @@ -834,27 +859,34 @@ row_vers_build_cur_vrow( return(cur_vrow); } -/*****************************************************************//** -Finds out if a version of the record, where the version >= the current +/** Finds out if a version of the record, where the version >= the current purge view, should have ientry as its secondary index entry. We check if there is any not delete marked version of the record where the trx -id >= purge view, and the secondary index entry and ientry are identified in -the alphabetical ordering; exactly in this case we return TRUE. +id >= purge view, and the secondary index entry == ientry; exactly in +this case we return TRUE. +@param[in] also_curr TRUE if also rec is included in the versions + to search; otherwise only versions prior + to it are searched +@param[in] rec record in the clustered index; the caller + must have a latch on the page +@param[in] mtr mtr holding the latch on rec; it will + also hold the latch on purge_view +@param[in] index secondary index +@param[in] ientry secondary index entry +@param[in] roll_ptr roll_ptr for the purge record +@param[in] trx_id transaction ID on the purging record +@param[in,out] vcol_info virtual column information for purge thread. @return TRUE if earlier version should have */ -ibool +bool row_vers_old_has_index_entry( -/*=========================*/ - ibool also_curr,/*!< in: TRUE if also rec is included in the - versions to search; otherwise only versions - prior to it are searched */ - const rec_t* rec, /*!< in: record in the clustered index; the - caller must have a latch on the page */ - mtr_t* mtr, /*!< in: mtr holding the latch on rec; it will - also hold the latch on purge_view */ - dict_index_t* index, /*!< in: the secondary index */ - const dtuple_t* ientry, /*!< in: the secondary index entry */ - roll_ptr_t roll_ptr,/*!< in: roll_ptr for the purge record */ - trx_id_t trx_id) /*!< in: transaction ID on the purging record */ + bool also_curr, + const rec_t* rec, + mtr_t* mtr, + dict_index_t* index, + const dtuple_t* ientry, + roll_ptr_t roll_ptr, + trx_id_t trx_id, + purge_vcol_info_t* vcol_info) { const rec_t* version; rec_t* prev_version; @@ -922,8 +954,18 @@ row_vers_old_has_index_entry( columns need to be computed */ if (trx_undo_roll_ptr_is_insert(t_roll_ptr) || dbug_v_purge) { + + if (vcol_info && !vcol_info->used) { + mtr->commit(); + } + row_vers_build_clust_v_col( - row, clust_index, index, heap); + row, clust_index, index, heap, + vcol_info); + + if (vcol_info && vcol_info->is_first_fetch()) { + goto unsafe_to_purge; + } entry = row_build_index_entry( row, ext, index, heap); @@ -988,7 +1030,7 @@ safe_to_purge: if (v_heap) { mem_heap_free(v_heap); } - return(TRUE); + return true; } } } else if (dict_index_has_virtual(index)) { @@ -996,9 +1038,15 @@ safe_to_purge: deleted, but the previous version of it might not. We will need to get the virtual column data from undo record associated with current cluster index */ + cur_vrow = row_vers_build_cur_vrow( also_curr, rec, clust_index, &clust_offsets, - index, ientry, roll_ptr, trx_id, heap, v_heap, mtr); + index, ientry, roll_ptr, trx_id, heap, v_heap, mtr, + vcol_info); + + if (vcol_info && vcol_info->is_first_fetch()) { + goto unsafe_to_purge; + } } version = rec; @@ -1017,14 +1065,14 @@ safe_to_purge: if (!prev_version) { /* Versions end here */ - +unsafe_to_purge: mem_heap_free(heap); if (v_heap) { mem_heap_free(v_heap); } - return(FALSE); + return false; } clust_offsets = rec_get_offsets(prev_version, clust_index, From 1cc1d0429da14a041a6240c6fce17e0d31cad8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 7 Jul 2018 11:34:26 +0300 Subject: [PATCH 14/29] MDEV-16664: Change the default to innodb_lock_schedule_algorithm=fcfs The parameter innodb_lock_schedule_algorithm was introduced in MariaDB Server 10.1.19, 10.2.13, 10.3.4 as part of MDEV-11039. In MariaDB 10.1, the default value of the parameter is 'fcfs', that is, the existing algorithm is used by default. But in later versions of MariaDB Server, the parameter was 'vats', enabling the new algorithm. Because the new algorithm is triggering a debug assertion failure that suggests corruption of the transactional lock data structures, we will revert to the old algorithm by default until we have resolved the problem. --- mysql-test/suite/sys_vars/r/sysvars_innodb.result | 4 ++-- storage/innobase/handler/ha_innodb.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 651461860a2..894f9a95ccb 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1604,9 +1604,9 @@ READ_ONLY YES COMMAND_LINE_ARGUMENT NONE VARIABLE_NAME INNODB_LOCK_SCHEDULE_ALGORITHM SESSION_VALUE NULL -GLOBAL_VALUE vats +GLOBAL_VALUE fcfs GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE vats +DEFAULT_VALUE fcfs VARIABLE_SCOPE GLOBAL VARIABLE_TYPE ENUM VARIABLE_COMMENT The algorithm Innodb uses for deciding which locks to grant next when a lock is released. Possible values are FCFS grant the locks in First-Come-First-Served order; VATS use the Variance-Aware-Transaction-Scheduling algorithm, which uses an Eldest-Transaction-First heuristic. diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e9d7a6d94df..deb6f05e853 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -20424,7 +20424,7 @@ static MYSQL_SYSVAR_ENUM(lock_schedule_algorithm, innodb_lock_schedule_algorithm " VATS" " use the Variance-Aware-Transaction-Scheduling algorithm, which" " uses an Eldest-Transaction-First heuristic.", - NULL, NULL, INNODB_LOCK_SCHEDULE_ALGORITHM_VATS, + NULL, NULL, INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, &innodb_lock_schedule_algorithm_typelib); static MYSQL_SYSVAR_ULONG(buffer_pool_instances, srv_buf_pool_instances, From 813b7398500f8d216c11da9b92135048a03f3227 Mon Sep 17 00:00:00 2001 From: Jacob Mathew Date: Mon, 9 Jul 2018 16:09:20 -0700 Subject: [PATCH 15/29] MDEV-16246: insert timestamp into spider table from mysqldump gets wrong time zone. The problem occurred because the Spider node was incorrectly handling timestamp values sent to and received from the data nodes. The problem has been corrected as follows: - Added logic to set and maintain the UTC time zone on the data nodes. To prevent timestamp ambiguity, it is necessary for the data nodes to use a time zone such as UTC which does not have daylight savings time. - Removed the spider_sync_time_zone configuration variable, which did not solve the problem and which interfered with the solution. - Added logic to convert to the UTC time zone all timestamp values sent to and received from the data nodes. This is done for both unique and non-unique timestamp columns. It is done for WHERE clauses, applying to SELECT, UPDATE and DELETE statements, and for UPDATE columns. - Disabled Spider's use of direct update when any of the columns to update is a timestamp column. This is necessary to prevent false duplicate key value errors. - Added a new test spider.timestamp to thoroughly test Spider's handling of timestamp values. Author: Jacob Mathew. Reviewer: Kentoku Shiba. Cherry-Picked: Commit 97cc9d3 on branch bb-10.3-MDEV-16246 --- sql/ha_partition.cc | 12 +- sql/ha_partition.h | 4 +- sql/handler.h | 4 +- sql/sql_update.cc | 5 +- storage/spider/ha_spider.cc | 120 +++-- storage/spider/ha_spider.h | 34 +- .../mysql-test/spider/r/timestamp.result | 370 ++++++++++++++ .../spider/mysql-test/spider/t/timestamp.test | 465 +++++++++++++++++ .../mysql-test/spider/t/timestamp_deinit.inc | 13 + .../mysql-test/spider/t/timestamp_init.inc | 45 ++ storage/spider/spd_conn.cc | 18 + storage/spider/spd_conn.h | 6 + storage/spider/spd_db_conn.cc | 479 ++++++++++++++---- storage/spider/spd_db_conn.h | 14 + storage/spider/spd_db_include.h | 1 + storage/spider/spd_db_mysql.cc | 240 ++++++--- storage/spider/spd_db_mysql.h | 2 + storage/spider/spd_db_oracle.h | 2 + storage/spider/spd_group_by_handler.cc | 24 +- storage/spider/spd_include.h | 2 + storage/spider/spd_param.cc | 21 - storage/spider/spd_param.h | 3 - storage/spider/spd_trx.cc | 5 +- 23 files changed, 1590 insertions(+), 299 deletions(-) create mode 100644 storage/spider/mysql-test/spider/r/timestamp.result create mode 100644 storage/spider/mysql-test/spider/t/timestamp.test create mode 100644 storage/spider/mysql-test/spider/t/timestamp_deinit.inc create mode 100644 storage/spider/mysql-test/spider/t/timestamp_init.inc diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index befcc62080e..7e04cabc765 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -11143,13 +11143,14 @@ int ha_partition::end_bulk_delete() SYNOPSIS direct_update_rows_init() + update fields Pointer to the list of fields to update RETURN VALUE >0 Error 0 Success */ -int ha_partition::direct_update_rows_init() +int ha_partition::direct_update_rows_init(List *update_fields) { int error; uint i, found; @@ -11175,8 +11176,8 @@ int ha_partition::direct_update_rows_init() { file= m_file[i]; if (unlikely((error= (m_pre_calling ? - file->pre_direct_update_rows_init() : - file->direct_update_rows_init())))) + file->pre_direct_update_rows_init(update_fields) : + file->direct_update_rows_init(update_fields))))) { DBUG_PRINT("info", ("partition FALSE by storage engine")); DBUG_RETURN(error); @@ -11214,20 +11215,21 @@ int ha_partition::direct_update_rows_init() SYNOPSIS pre_direct_update_rows_init() + update fields Pointer to the list of fields to update RETURN VALUE >0 Error 0 Success */ -int ha_partition::pre_direct_update_rows_init() +int ha_partition::pre_direct_update_rows_init(List *update_fields) { bool save_m_pre_calling; int error; DBUG_ENTER("ha_partition::pre_direct_update_rows_init"); save_m_pre_calling= m_pre_calling; m_pre_calling= TRUE; - error= direct_update_rows_init(); + error= direct_update_rows_init(update_fields); m_pre_calling= save_m_pre_calling; DBUG_RETURN(error); } diff --git a/sql/ha_partition.h b/sql/ha_partition.h index e661d0badd3..8a251016703 100644 --- a/sql/ha_partition.h +++ b/sql/ha_partition.h @@ -620,8 +620,8 @@ public: virtual int bulk_update_row(const uchar *old_data, const uchar *new_data, ha_rows *dup_key_found); virtual int update_row(const uchar * old_data, const uchar * new_data); - virtual int direct_update_rows_init(); - virtual int pre_direct_update_rows_init(); + virtual int direct_update_rows_init(List *update_fields); + virtual int pre_direct_update_rows_init(List *update_fields); virtual int direct_update_rows(ha_rows *update_rows); virtual int pre_direct_update_rows(); virtual bool start_bulk_delete(); diff --git a/sql/handler.h b/sql/handler.h index 54df7941526..ce8711bd5ab 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -4407,12 +4407,12 @@ private: /* Perform initialization for a direct update request */ public: int ha_direct_update_rows(ha_rows *update_rows); - virtual int direct_update_rows_init() + virtual int direct_update_rows_init(List *update_fields) { return HA_ERR_WRONG_COMMAND; } private: - virtual int pre_direct_update_rows_init() + virtual int pre_direct_update_rows_init(List *update_fields) { return HA_ERR_WRONG_COMMAND; } diff --git a/sql/sql_update.cc b/sql/sql_update.cc index c0fa2b4d7fe..cb7bcdc33a1 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -615,6 +615,9 @@ int mysql_update(THD *thd, - Note that Spider can handle ORDER BY and LIMIT in a cluster with one data node. These conditions are therefore checked in direct_update_rows_init(). + - Update fields include a unique timestamp field + - The storage engine may not be able to avoid false duplicate key + errors. This condition is checked in direct_update_rows_init(). Direct update does not require a WHERE clause @@ -637,7 +640,7 @@ int mysql_update(THD *thd, if (!table->file->info_push(INFO_KIND_UPDATE_FIELDS, &fields) && !table->file->info_push(INFO_KIND_UPDATE_VALUES, &values) && - !table->file->direct_update_rows_init()) + !table->file->direct_update_rows_init(&fields)) { do_direct_update= TRUE; diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc index 9668e4cfbdb..bd302f155d2 100644 --- a/storage/spider/ha_spider.cc +++ b/storage/spider/ha_spider.cc @@ -10064,13 +10064,11 @@ int ha_spider::update_row( #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS_WITH_HS -int ha_spider::direct_update_rows_init( - uint mode, - KEY_MULTI_RANGE *ranges, - uint range_count, - bool sorted, - const uchar *new_data -) { +int ha_spider::direct_update_rows_init(List *update_fields, uint mode, + KEY_MULTI_RANGE *ranges, + uint range_count, bool sorted, + const uchar *new_data) +{ #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) int error_num; #endif @@ -10098,7 +10096,7 @@ int ha_spider::direct_update_rows_init( DBUG_RETURN(pre_direct_init_result); } DBUG_RETURN(bulk_access_link_exec_tgt->spider->direct_update_rows_init( - mode, ranges, range_count, sorted, new_data)); + update_fields, mode, ranges, range_count, sorted, new_data)); } #endif direct_update_init( @@ -10202,14 +10200,46 @@ int ha_spider::direct_update_rows_init( DBUG_RETURN(HA_ERR_WRONG_COMMAND); } #else -int ha_spider::direct_update_rows_init() +/** + Perform initialization for a direct update request. + + @param update fields Pointer to the list of fields to update. + + @return >0 Error. + 0 Success. +*/ + +int ha_spider::direct_update_rows_init(List *update_fields) { st_select_lex *select_lex; longlong select_limit; longlong offset_limit; + List_iterator it(*update_fields); + Item *item; + Field *field; THD *thd = trx->thd; DBUG_ENTER("ha_spider::direct_update_rows_init"); DBUG_PRINT("info",("spider this=%p", this)); + + while ((item = it++)) + { + if (item->type() == Item::FIELD_ITEM) + { + field = ((Item_field *)item)->field; + + if (field->type() == FIELD_TYPE_TIMESTAMP && + field->flags & UNIQUE_KEY_FLAG) + { + /* + Spider cannot perform direct update on unique timestamp fields. + To avoid false duplicate key errors, the table needs to be + updated one row at a time. + */ + DBUG_RETURN(HA_ERR_WRONG_COMMAND); + } + } + } + #ifdef HA_CAN_BULK_ACCESS if ( bulk_access_executing && @@ -10227,7 +10257,8 @@ int ha_spider::direct_update_rows_init() pre_direct_init_result)); DBUG_RETURN(pre_direct_init_result); } - DBUG_RETURN(bulk_access_link_exec_tgt->spider->direct_update_rows_init()); + DBUG_RETURN(bulk_access_link_exec_tgt->spider-> + direct_update_rows_init(List *update_fields)); } #endif direct_update_init( @@ -10298,31 +10329,11 @@ int ha_spider::direct_update_rows_init() #ifdef HA_CAN_BULK_ACCESS #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS_WITH_HS -int ha_spider::pre_direct_update_rows_init( - uint mode, - KEY_MULTI_RANGE *ranges, - uint range_count, - bool sorted, - const uchar *new_data -) { - int error_num; - DBUG_ENTER("ha_spider::pre_direct_update_rows_init"); - DBUG_PRINT("info",("spider this=%p", this)); - if (bulk_access_started) - { - error_num = bulk_access_link_current->spider-> - pre_direct_update_rows_init( - mode, ranges, range_count, sorted, new_data); - bulk_access_link_current->spider->bulk_access_pre_called = TRUE; - bulk_access_link_current->called = TRUE; - DBUG_RETURN(error_num); - } - pre_direct_init_result = direct_update_rows_init( - mode, ranges, range_count, sorted, new_data); - DBUG_RETURN(pre_direct_init_result); -} -#else -int ha_spider::pre_direct_update_rows_init() +int ha_spider::pre_direct_update_rows_init(List *update_fields, + uint mode, + KEY_MULTI_RANGE *ranges, + uint range_count, bool sorted, + const uchar *new_data) { int error_num; DBUG_ENTER("ha_spider::pre_direct_update_rows_init"); @@ -10330,12 +10341,42 @@ int ha_spider::pre_direct_update_rows_init() if (bulk_access_started) { error_num = bulk_access_link_current->spider-> - pre_direct_update_rows_init(); + pre_direct_update_rows_init(update_fields, mode, ranges, range_count, + sorted, new_data); bulk_access_link_current->spider->bulk_access_pre_called = TRUE; bulk_access_link_current->called = TRUE; DBUG_RETURN(error_num); } - pre_direct_init_result = direct_update_rows_init(); + pre_direct_init_result = direct_update_rows_init(update_fields, mode, + ranges, range_count, + sorted, new_data); + DBUG_RETURN(pre_direct_init_result); +} +#else +/** + Do initialization for performing parallel direct update + for a handlersocket update request. + + @param update fields Pointer to the list of fields to update. + + @return >0 Error. + 0 Success. +*/ + +int ha_spider::pre_direct_update_rows_init(List *update_fields) +{ + int error_num; + DBUG_ENTER("ha_spider::pre_direct_update_rows_init"); + DBUG_PRINT("info",("spider this=%p", this)); + if (bulk_access_started) + { + error_num = bulk_access_link_current->spider-> + pre_direct_update_rows_init(update_fields); + bulk_access_link_current->spider->bulk_access_pre_called = TRUE; + bulk_access_link_current->called = TRUE; + DBUG_RETURN(error_num); + } + pre_direct_init_result = direct_update_rows_init(update_fields); DBUG_RETURN(pre_direct_init_result); } #endif @@ -15733,8 +15774,9 @@ int ha_spider::print_item_type( dbton_hdl = dbton_handler[dbton_id]; if ( dbton_hdl->first_link_idx >= 0 && - (error_num = spider_db_print_item_type(item, this, str, - alias, alias_length, dbton_id, FALSE, NULL)) + (error_num = spider_db_print_item_type(item, NULL, this, str, + alias, alias_length, dbton_id, + FALSE, NULL)) ) { DBUG_RETURN(error_num); } diff --git a/storage/spider/ha_spider.h b/storage/spider/ha_spider.h index b79e1b89fbf..66e5cf8a452 100644 --- a/storage/spider/ha_spider.h +++ b/storage/spider/ha_spider.h @@ -587,35 +587,29 @@ public: ); #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS_WITH_HS - inline int direct_update_rows_init() + inline int direct_update_rows_init(List *update_fields) { - return direct_update_rows_init(2, NULL, 0, FALSE, NULL); + return direct_update_rows_init(update_fields, 2, NULL, 0, FALSE, NULL); } - int direct_update_rows_init( - uint mode, - KEY_MULTI_RANGE *ranges, - uint range_count, - bool sorted, - const uchar *new_data - ); + int direct_update_rows_init(List *update_fields, uint mode, + KEY_MULTI_RANGE *ranges, uint range_count, + bool sorted, const uchar *new_data); #else - int direct_update_rows_init(); + int direct_update_rows_init(List *update_fields); #endif #ifdef HA_CAN_BULK_ACCESS #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS_WITH_HS - inline int pre_direct_update_rows_init() + inline int pre_direct_update_rows_init(List *update_fields) { - return pre_direct_update_rows_init(2, NULL, 0, FALSE, NULL); + return pre_direct_update_rows_init(update_fields, + 2, NULL, 0, FALSE, NULL); } - int pre_direct_update_rows_init( - uint mode, - KEY_MULTI_RANGE *ranges, - uint range_count, - bool sorted, - uchar *new_data - ); + int pre_direct_update_rows_init(List *update_fields, + uint mode, KEY_MULTI_RANGE *ranges, + uint range_count, bool sorted, + uchar *new_data); #else - int pre_direct_update_rows_init(); + int pre_direct_update_rows_init(List *update_fields); #endif #endif #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS_WITH_HS diff --git a/storage/spider/mysql-test/spider/r/timestamp.result b/storage/spider/mysql-test/spider/r/timestamp.result new file mode 100644 index 00000000000..df40549cab2 --- /dev/null +++ b/storage/spider/mysql-test/spider/r/timestamp.result @@ -0,0 +1,370 @@ +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +child3_1 +child3_2 +child3_3 + +Initialize Time Zone +connection master_1; +SET GLOBAL time_zone='MET'; +SET time_zone='MET'; +connection child2_1; +SET GLOBAL time_zone='MET'; +SET time_zone='MET'; + +drop and create databases +connection master_1; +DROP DATABASE IF EXISTS ts_test_local; +CREATE DATABASE ts_test_local; +USE ts_test_local; +connection child2_1; +SET @old_log_output = @@global.log_output; +SET GLOBAL log_output = 'TABLE,FILE'; +DROP DATABASE IF EXISTS ts_test_remote; +CREATE DATABASE ts_test_remote; +USE ts_test_remote; + +test select 1 +connection master_1; +SELECT 1; +1 +1 +connection child2_1; +SELECT 1; +1 +1 + +create table +connection child2_1; +CHILD2_1_DROP_TABLES +CHILD2_1_CREATE_TABLES +TRUNCATE TABLE mysql.general_log; +connection master_1; +DROP TABLE IF EXISTS tbl_a; +CREATE TABLE tbl_a ( +col_a INT UNSIGNED NOT NULL AUTO_INCREMENT, +col_dt DATETIME, +col_ts TIMESTAMP NOT NULL +DEFAULT current_timestamp() ON UPDATE current_timestamp(), +PRIMARY KEY(col_a), +) MASTER_1_ENGINE MASTER_1_AUTO_INCREMENT_2_1 MASTER_1_COMMENT_2_1 +SHOW CREATE TABLE tbl_a; +Table Create Table +tbl_a CREATE TABLE `tbl_a` ( + `col_a` int(10) unsigned NOT NULL AUTO_INCREMENT, + `col_dt` datetime DEFAULT NULL, + `col_ts` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), + PRIMARY KEY (`col_a`), + UNIQUE KEY `i_ts` (`col_ts`) +) ENGINE=SPIDER AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 COMMENT='database "ts_test_remote", table "tbl_a", srv "s_2_1"' + +Set a different time zone that has DST +SET time_zone='+01:00'; + +Insert Rows +connection master_1; +Min value +SET @@timestamp=1; +INSERT INTO tbl_a VALUES (1, now(), now()); +SET @@timestamp=0; +Ambiguous DST values for MET time zone that result in the +same UTC timestamp +INSERT INTO tbl_a VALUES (2, '2018-03-25 02:00:00', '2018-03-25 02:00:00'); +INSERT INTO tbl_a VALUES (3, '2018-03-25 02:30:00', '2018-03-25 02:30:00'); +Ambiguous DST values for MET time zone in the 2:00 am to 3:00 am hour +that occur twice when transitioning from DST to standard time +SET @@timestamp=1540686600; +INSERT INTO tbl_a VALUES (4, now(), now()); +SET @@timestamp=1540690200; +INSERT INTO tbl_a VALUES (5, now(), now()); +Max value +SET @@timestamp=2147483647; +INSERT INTO tbl_a VALUES (6, now(), now()); +SET @@timestamp=0; + +SELECTs +connection child2_1; +TRUNCATE TABLE mysql.general_log; +connection master_1; +SELECT *, unix_timestamp(col_ts) FROM tbl_a; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; +argument +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %' +SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 03:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 03:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 02:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 + +DELETEs +connection child2_1; +TRUNCATE TABLE mysql.general_log; +connection master_1; +DELETE FROM tbl_a WHERE col_ts='1970-01-01 01:00:01'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a; +col_a col_dt col_ts unix_timestamp(col_ts) +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; +argument +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %' +SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a; +col_a col_dt col_ts unix_timestamp(col_ts) +2 2018-03-25 02:00:00 2018-03-25 03:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 03:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 02:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +connection child2_1; +TRUNCATE TABLE mysql.general_log; +connection master_1; +SET @@timestamp=1; +INSERT INTO tbl_a VALUES (1, now(), now()); +SET @@timestamp=0; +SELECT *, unix_timestamp(col_ts) FROM tbl_a; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; +argument +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %' +SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 03:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 03:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 02:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 + +UPDATEs +connection child2_1; +TRUNCATE TABLE mysql.general_log; +connection master_1; +UPDATE tbl_a SET col_ts=col_dt; +SELECT *, unix_timestamp(col_ts) FROM tbl_a; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; +argument +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` for update +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %' +SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 03:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 03:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 02:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 + +Lookups +connection child2_1; +TRUNCATE TABLE mysql.general_log; +connection master_1; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > '2018-01-01'; +col_a col_dt col_ts unix_timestamp(col_ts) +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts < '2018-10-28 02:30:00'; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE '2018-10-28 02:30:00' > col_ts; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts BETWEEN '2018-10-28 01:30:00' AND '2018-10-28 02:30:00'; +col_a col_dt col_ts unix_timestamp(col_ts) +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts >= '2018-10-28 01:30:00' AND col_ts <= '2018-10-28 02:30:00'; +col_a col_dt col_ts unix_timestamp(col_ts) +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 180325020000; +col_a col_dt col_ts unix_timestamp(col_ts) +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 19700101010001; +col_a col_dt col_ts unix_timestamp(col_ts) +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; +argument +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '2017-12-31 23:00:00') order by `col_ts` +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` < '2018-10-28 01:30:00') order by `col_ts` +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where ('2018-10-28 01:30:00' > `col_ts`) order by `col_ts` +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where `col_ts` >= '2018-10-28 00:30:00' and `col_ts` <= '2018-10-28 01:30:00' and (`col_ts` between '2018-10-28 00:30:00' and '2018-10-28 01:30:00') order by `col_ts` +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where `col_ts` >= '2018-10-28 00:30:00' and `col_ts` <= '2018-10-28 01:30:00' and ((`col_ts` >= '2018-10-28 00:30:00') and (`col_ts` <= '2018-10-28 01:30:00')) order by `col_ts` +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '2018-03-25 01:00:00') order by `col_ts` +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '1970-01-01 00:00:01') order by `col_ts` +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %' +SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 03:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 03:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 02:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 + +Drop the index on the timestamp column +connection child2_1; +DROP INDEX i_ts ON tbl_a; +SHOW CREATE TABLE tbl_a; +Table Create Table +tbl_a CREATE TABLE `tbl_a` ( + `col_a` int(10) unsigned NOT NULL AUTO_INCREMENT, + `col_dt` datetime DEFAULT NULL, + `col_ts` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), + PRIMARY KEY (`col_a`) +) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 +TRUNCATE TABLE mysql.general_log; +connection master_1; +DROP INDEX i_ts ON tbl_a; +SHOW CREATE TABLE tbl_a; +Table Create Table +tbl_a CREATE TABLE `tbl_a` ( + `col_a` int(10) unsigned NOT NULL AUTO_INCREMENT, + `col_dt` datetime DEFAULT NULL, + `col_ts` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), + PRIMARY KEY (`col_a`) +) ENGINE=SPIDER AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 COMMENT='database "ts_test_remote", table "tbl_a", srv "s_2_1"' + +Retry lookups on unindexed timestamp column +connection child2_1; +TRUNCATE TABLE mysql.general_log; +connection master_1; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > '2018-01-01'; +col_a col_dt col_ts unix_timestamp(col_ts) +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts < '2018-10-28 02:30:00'; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE '2018-10-28 02:30:00' > col_ts; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts BETWEEN '2018-10-28 01:30:00' AND '2018-10-28 02:30:00'; +col_a col_dt col_ts unix_timestamp(col_ts) +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts >= '2018-10-28 01:30:00' AND col_ts <= '2018-10-28 02:30:00'; +col_a col_dt col_ts unix_timestamp(col_ts) +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 180325020000; +col_a col_dt col_ts unix_timestamp(col_ts) +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 19700101010001; +col_a col_dt col_ts unix_timestamp(col_ts) +2 2018-03-25 02:00:00 2018-03-25 02:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 02:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 01:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 +connection child2_1; +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; +argument +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '2017-12-31 23:00:00') +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` < '2018-10-28 01:30:00') +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where ('2018-10-28 01:30:00' > `col_ts`) +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` between '2018-10-28 00:30:00' and '2018-10-28 01:30:00') +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where ((`col_ts` >= '2018-10-28 00:30:00') and (`col_ts` <= '2018-10-28 01:30:00')) +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '2018-03-25 01:00:00') +select `col_a`,`col_dt`,`col_ts` from `ts_test_remote`.`tbl_a` where (`col_ts` > '1970-01-01 00:00:01') +SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %' +SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a; +col_a col_dt col_ts unix_timestamp(col_ts) +1 1970-01-01 01:00:01 1970-01-01 01:00:01 1 +2 2018-03-25 02:00:00 2018-03-25 03:00:00 1521939600 +3 2018-03-25 02:30:00 2018-03-25 03:30:00 1521941400 +4 2018-10-28 01:30:00 2018-10-28 02:30:00 1540686600 +5 2018-10-28 02:30:00 2018-10-28 02:30:00 1540690200 +6 2038-01-19 04:14:07 2038-01-19 04:14:07 2147483647 + +Restore Time Zone settings +connection master_1; +SET GLOBAL time_zone=DEFAULT; +SET time_zone=DEFAULT; +connection child2_1; +SET GLOBAL time_zone=DEFAULT; +SET time_zone=DEFAULT; + +deinit +connection master_1; +DROP DATABASE IF EXISTS ts_test_local; +connection child2_1; +DROP DATABASE IF EXISTS ts_test_remote; +SET GLOBAL log_output = @old_log_output; +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +child3_1 +child3_2 +child3_3 + +end of test diff --git a/storage/spider/mysql-test/spider/t/timestamp.test b/storage/spider/mysql-test/spider/t/timestamp.test new file mode 100644 index 00000000000..7008638cc4a --- /dev/null +++ b/storage/spider/mysql-test/spider/t/timestamp.test @@ -0,0 +1,465 @@ +--source timestamp_init.inc + +--echo +--echo Initialize Time Zone +--connection master_1 +SET GLOBAL time_zone='MET'; +SET time_zone='MET'; +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + SET GLOBAL time_zone='MET'; + SET time_zone='MET'; +} + +--echo +--echo drop and create databases +--connection master_1 +--disable_warnings +DROP DATABASE IF EXISTS ts_test_local; +CREATE DATABASE ts_test_local; +USE ts_test_local; +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + if ($USE_GENERAL_LOG) + { + SET @old_log_output = @@global.log_output; + SET GLOBAL log_output = 'TABLE,FILE'; + } + DROP DATABASE IF EXISTS ts_test_remote; + CREATE DATABASE ts_test_remote; + USE ts_test_remote; +} +--enable_warnings + +--echo +--echo test select 1 +--connection master_1 +SELECT 1; +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + SELECT 1; + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} + +--echo +--echo create table +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($OUTPUT_CHILD_GROUP2) + { + --disable_query_log + echo CHILD2_1_DROP_TABLES; + echo CHILD2_1_CREATE_TABLES; + } + --disable_warnings + eval $CHILD2_1_DROP_TABLES; + --enable_warnings + eval $CHILD2_1_CREATE_TABLES; + if ($OUTPUT_CHILD_GROUP2) + { + --enable_query_log + } + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +--connection master_1 +--disable_warnings +DROP TABLE IF EXISTS tbl_a; +--enable_warnings +--disable_query_log +echo CREATE TABLE tbl_a ( + col_a INT UNSIGNED NOT NULL AUTO_INCREMENT, + col_dt DATETIME, + col_ts TIMESTAMP NOT NULL + DEFAULT current_timestamp() ON UPDATE current_timestamp(), + PRIMARY KEY(col_a), + +) MASTER_1_ENGINE MASTER_1_AUTO_INCREMENT_2_1 MASTER_1_COMMENT_2_1; +eval CREATE TABLE tbl_a ( + col_a INT UNSIGNED NOT NULL AUTO_INCREMENT, + col_dt DATETIME, + col_ts TIMESTAMP NOT NULL + DEFAULT current_timestamp() ON UPDATE current_timestamp(), + PRIMARY KEY(col_a), + UNIQUE INDEX i_ts (col_ts) +) $MASTER_1_ENGINE $MASTER_1_AUTO_INCREMENT_2_1 $MASTER_1_COMMENT_2_1; +--enable_query_log +SHOW CREATE TABLE tbl_a; + +--echo +--echo Set a different time zone that has DST +SET time_zone='+01:00'; + +--echo +--echo Insert Rows +--connection master_1 +--echo Min value +SET @@timestamp=1; +INSERT INTO tbl_a VALUES (1, now(), now()); +SET @@timestamp=0; +--echo Ambiguous DST values for MET time zone that result in the +--echo same UTC timestamp +INSERT INTO tbl_a VALUES (2, '2018-03-25 02:00:00', '2018-03-25 02:00:00'); +INSERT INTO tbl_a VALUES (3, '2018-03-25 02:30:00', '2018-03-25 02:30:00'); +--echo Ambiguous DST values for MET time zone in the 2:00 am to 3:00 am hour +--echo that occur twice when transitioning from DST to standard time +SET @@timestamp=1540686600; +INSERT INTO tbl_a VALUES (4, now(), now()); +SET @@timestamp=1540690200; +INSERT INTO tbl_a VALUES (5, now(), now()); +--echo Max value +SET @@timestamp=2147483647; +INSERT INTO tbl_a VALUES (6, now(), now()); +SET @@timestamp=0; + +--echo +--echo SELECTs +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +--connection master_1 +SELECT *, unix_timestamp(col_ts) FROM tbl_a; +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + eval $CHILD2_1_SELECT_ARGUMENT1; + } + eval $CHILD2_1_SELECT_TABLES; + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} + +--echo +--echo DELETEs +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +--connection master_1 +DELETE FROM tbl_a WHERE col_ts='1970-01-01 01:00:01'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a; +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + eval $CHILD2_1_SELECT_ARGUMENT1; + } + eval $CHILD2_1_SELECT_TABLES; + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +--connection master_1 +SET @@timestamp=1; +INSERT INTO tbl_a VALUES (1, now(), now()); +SET @@timestamp=0; +SELECT *, unix_timestamp(col_ts) FROM tbl_a; +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + eval $CHILD2_1_SELECT_ARGUMENT1; + } + eval $CHILD2_1_SELECT_TABLES; + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} + +--echo +--echo UPDATEs +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +--connection master_1 +UPDATE tbl_a SET col_ts=col_dt; +SELECT *, unix_timestamp(col_ts) FROM tbl_a; +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + eval $CHILD2_1_SELECT_ARGUMENT1; + } + eval $CHILD2_1_SELECT_TABLES; + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} + +--echo +--echo Lookups +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +--connection master_1 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > '2018-01-01'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts < '2018-10-28 02:30:00'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE '2018-10-28 02:30:00' > col_ts; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts BETWEEN '2018-10-28 01:30:00' AND '2018-10-28 02:30:00'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts >= '2018-10-28 01:30:00' AND col_ts <= '2018-10-28 02:30:00'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 180325020000; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 19700101010001; +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + eval $CHILD2_1_SELECT_ARGUMENT1; + } + eval $CHILD2_1_SELECT_TABLES; + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} + +--echo +--echo Drop the index on the timestamp column +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + eval $CHILD2_1_DROP_INDEX; + eval $CHILD2_1_SHOW_CREATE_TABLE; + if ($OUTPUT_CHILD_GROUP2) + { + --enable_query_log + } + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +--connection master_1 +DROP INDEX i_ts ON tbl_a; +SHOW CREATE TABLE tbl_a; + +--echo +--echo Retry lookups on unindexed timestamp column +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} +--connection master_1 +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > '2018-01-01'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts < '2018-10-28 02:30:00'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE '2018-10-28 02:30:00' > col_ts; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts BETWEEN '2018-10-28 01:30:00' AND '2018-10-28 02:30:00'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts >= '2018-10-28 01:30:00' AND col_ts <= '2018-10-28 02:30:00'; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 180325020000; +SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 19700101010001; +if ($USE_CHILD_GROUP2) +{ + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_1 + if ($USE_GENERAL_LOG) + { + eval $CHILD2_1_SELECT_ARGUMENT1; + } + eval $CHILD2_1_SELECT_TABLES; + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } +} + +--echo +--echo Restore Time Zone settings +--connection master_1 +SET GLOBAL time_zone=DEFAULT; +SET time_zone=DEFAULT; +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + SET GLOBAL time_zone=DEFAULT; + SET time_zone=DEFAULT; +} + +--echo +--echo deinit +--disable_warnings +--connection master_1 +DROP DATABASE IF EXISTS ts_test_local; +if ($USE_CHILD_GROUP2) +{ + --connection child2_1 + DROP DATABASE IF EXISTS ts_test_remote; + if ($USE_GENERAL_LOG) + { + SET GLOBAL log_output = @old_log_output; + } +} +--enable_warnings +--source timestamp_deinit.inc +--echo +--echo end of test diff --git a/storage/spider/mysql-test/spider/t/timestamp_deinit.inc b/storage/spider/mysql-test/spider/t/timestamp_deinit.inc new file mode 100644 index 00000000000..52be67a1d09 --- /dev/null +++ b/storage/spider/mysql-test/spider/t/timestamp_deinit.inc @@ -0,0 +1,13 @@ +--let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP +--let $CHILD2_1_DROP_TABLES= $CHILD2_1_DROP_TABLES_BACKUP +--let $CHILD2_1_CREATE_TABLES= $CHILD2_1_CREATE_TABLES_BACKUP +--let $CHILD2_1_SELECT_TABLES= $CHILD2_1_SELECT_TABLES_BACKUP +--let $OUTPUT_CHILD_GROUP2= $OUTPUT_CHILD_GROUP2_BACKUP +--let $USE_GENERAL_LOG= $USE_GENERAL_LOG_BACKUP +--disable_warnings +--disable_query_log +--disable_result_log +--source ../t/test_deinit.inc +--enable_result_log +--enable_query_log +--enable_warnings diff --git a/storage/spider/mysql-test/spider/t/timestamp_init.inc b/storage/spider/mysql-test/spider/t/timestamp_init.inc new file mode 100644 index 00000000000..12049ab34fa --- /dev/null +++ b/storage/spider/mysql-test/spider/t/timestamp_init.inc @@ -0,0 +1,45 @@ +--disable_warnings +--disable_query_log +--disable_result_log +--source ../t/test_init.inc +--enable_result_log +--enable_query_log +--enable_warnings + +--let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1 +let $MASTER_1_COMMENT_2_1= + COMMENT='database "ts_test_remote", table "tbl_a", srv "s_2_1"'; +let $MASTER_1_AUTO_INCREMENT_2_1= + AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4; +let $MASTER_1_AUTO_INCREMENT1= + AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4; +let $MASTER_1_AUTO_INCREMENT2= + AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4; +let $CHILD2_1_AUTO_INCREMENT= + AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4; +--let $CHILD2_1_DROP_TABLES_BACKUP= $CHILD2_1_DROP_TABLES +let $CHILD2_1_DROP_TABLES= + DROP TABLE IF EXISTS tbl_a; +--let $CHILD2_1_CREATE_TABLES_BACKUP= $CHILD2_1_CREATE_TABLES +let $CHILD2_1_CREATE_TABLES= + CREATE TABLE tbl_a ( + col_a INT UNSIGNED NOT NULL AUTO_INCREMENT, + col_dt DATETIME, + col_ts TIMESTAMP NOT NULL + DEFAULT current_timestamp() ON UPDATE current_timestamp(), + PRIMARY KEY(col_a), + UNIQUE INDEX i_ts (col_ts) + ) $CHILD2_1_ENGINE $CHILD2_1_AUTO_INCREMENT; +--let $CHILD2_1_SELECT_TABLES_BACKUP= $CHILD2_1_SELECT_TABLES +let $CHILD2_1_SELECT_TABLES= + SELECT col_a, col_dt, col_ts, unix_timestamp(col_ts) FROM tbl_a ORDER BY col_a; +let $CHILD2_1_SELECT_ARGUMENT1= + SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; +let $CHILD2_1_DROP_INDEX= + DROP INDEX i_ts ON tbl_a; +let $CHILD2_1_SHOW_CREATE_TABLE= + SHOW CREATE TABLE tbl_a; +--let $OUTPUT_CHILD_GROUP2_BACKUP= $OUTPUT_CHILD_GROUP2 +--let $OUTPUT_CHILD_GROUP2= 1 +--let $USE_GENERAL_LOG_BACKUP= $USE_GENERAL_LOG +--let $USE_GENERAL_LOG= 1 diff --git a/storage/spider/spd_conn.cc b/storage/spider/spd_conn.cc index 0ca7555385c..148ad43d337 100644 --- a/storage/spider/spd_conn.cc +++ b/storage/spider/spd_conn.cc @@ -88,6 +88,9 @@ extern PSI_thread_key spd_key_thd_bg_mon; #endif #endif +/* UTC time zone for timestamp columns */ +extern Time_zone *UTC; + HASH spider_open_connections; uint spider_open_connections_id; HASH spider_ipport_conns; @@ -454,6 +457,13 @@ SPIDER_CONN *spider_create_conn( char *tmp_ssl_cipher, *tmp_ssl_key, *tmp_default_file, *tmp_default_group; DBUG_ENTER("spider_create_conn"); + if (unlikely(!UTC)) + { + /* UTC time zone for timestamp columns */ + String tz_00_name(STRING_WITH_LEN("+00:00"), &my_charset_bin); + UTC = my_tz_find(current_thd, &tz_00_name); + } + #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) if (conn_kind == SPIDER_CONN_KIND_MYSQL) { @@ -1429,6 +1439,14 @@ void spider_conn_queue_time_zone( DBUG_VOID_RETURN; } +void spider_conn_queue_UTC_time_zone(SPIDER_CONN *conn) +{ + DBUG_ENTER("spider_conn_queue_time_zone"); + DBUG_PRINT("info", ("spider conn=%p", conn)); + spider_conn_queue_time_zone(conn, UTC); + DBUG_VOID_RETURN; +} + void spider_conn_queue_start_transaction( SPIDER_CONN *conn ) { diff --git a/storage/spider/spd_conn.h b/storage/spider/spd_conn.h index 48c9d206d97..998658c5353 100644 --- a/storage/spider/spd_conn.h +++ b/storage/spider/spd_conn.h @@ -13,6 +13,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include "tztime.h" + #define SPIDER_LOCK_MODE_NO_LOCK 0 #define SPIDER_LOCK_MODE_SHARED 1 #define SPIDER_LOCK_MODE_EXCLUSIVE 2 @@ -137,6 +139,10 @@ void spider_conn_queue_time_zone( Time_zone *time_zone ); +void spider_conn_queue_UTC_time_zone( + SPIDER_CONN *conn +); + void spider_conn_queue_start_transaction( SPIDER_CONN *conn ); diff --git a/storage/spider/spd_db_conn.cc b/storage/spider/spd_db_conn.cc index 83de99e0d2f..ac5701a8274 100644 --- a/storage/spider/spd_db_conn.cc +++ b/storage/spider/spd_db_conn.cc @@ -76,6 +76,9 @@ extern HASH spider_open_connections; pthread_mutex_t spider_open_conn_mutex; const char spider_dig_upper[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; +/* UTC time zone for timestamp columns */ +extern Time_zone *UTC; + int spider_db_connect( const SPIDER_SHARE *share, SPIDER_CONN *conn, @@ -191,10 +194,14 @@ int spider_db_connect( } DBUG_RETURN(error_num); } + conn->connect_error = 0; conn->opened_handlers = 0; conn->db_conn->reset_opened_handler(); ++conn->connection_id; + + /* Set the connection's time zone to UTC */ + spider_conn_queue_UTC_time_zone(conn); DBUG_RETURN(0); } @@ -2866,12 +2873,20 @@ int spider_db_fetch_row( my_ptrdiff_t ptr_diff ) { int error_num; + THD *thd = field->table->in_use; + Time_zone *saved_time_zone = thd->variables.time_zone; DBUG_ENTER("spider_db_fetch_row"); DBUG_PRINT("info", ("spider field_name %s", field->field_name.str)); DBUG_PRINT("info", ("spider fieldcharset %s", field->charset()->csname)); + + thd->variables.time_zone = UTC; + field->move_field_offset(ptr_diff); error_num = row->store_to_field(field, share->access_charset); field->move_field_offset(-ptr_diff); + + thd->variables.time_zone = saved_time_zone; + DBUG_RETURN(error_num); } @@ -8449,18 +8464,80 @@ int spider_db_flush_logs( DBUG_RETURN(0); } -int spider_db_print_item_type( - Item *item, - ha_spider *spider, - spider_string *str, - const char *alias, - uint alias_length, - uint dbton_id, - bool use_fields, - spider_fields *fields -) { +/** + Find the field among the items in an expression tree. + + @param item_list List of items of the expression. + @param item_count Number of items in the item list. + @param start_item Index of the first item to consider. + @param str String into which the expression is to be printed. + @param func_name Function or operator name. + @param func_name_length Length of function or operator name. + + @return Pointer to the field in the item list if the list + contains only one field; NULL otherwise. +*/ + +Field *spider_db_find_field_in_item_list(Item **item_list, uint item_count, + uint start_item, spider_string *str, + const char *func_name, + int func_name_length) +{ + uint item_num; + Item *item; + Field *field = NULL; + DBUG_ENTER("spider_db_find_field_in_item_list"); + + if (str && func_name_length) + { + if (strncasecmp(func_name, ",", 1)) + { + /* A known function or operator */ + for (item_num = start_item; item_num < item_count; item_num++) + { + item = item_list[item_num]; + + if (item->type() == Item::FIELD_ITEM) + { + if (field) + { + /* Field is not relevant if there are multiple fields */ + DBUG_RETURN(NULL); + } + + field = ((Item_field *) item)->field; + } + } + } + } + + DBUG_RETURN(field); +} + +/** + Print an operand value within a statement generated for an expression. + + @param item Operand value to print. + @param field Field related to the operand value. + @param spider Spider. + @param str String into which the value is to be printed. + @param alias Name related to the operand. + @param alias_length Length of the name. + @param dbton_id Spider Db/Table id. + @param use_fields Use fields or exchange fields. + @param fields Array of fields in the expression. + + @return Error code. +*/ + +int spider_db_print_item_type(Item *item, Field *field, ha_spider *spider, + spider_string *str, const char *alias, + uint alias_length, uint dbton_id, + bool use_fields, spider_fields *fields) +{ DBUG_ENTER("spider_db_print_item_type"); DBUG_PRINT("info",("spider COND type=%d", item->type())); + switch (item->type()) { case Item::FUNC_ITEM: @@ -8484,19 +8561,25 @@ int spider_db_print_item_type( DBUG_RETURN(spider_db_open_item_row((Item_row *) item, spider, str, alias, alias_length, dbton_id, use_fields, fields)); case Item::STRING_ITEM: - DBUG_RETURN(spider_db_open_item_string(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_string(item, field, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); case Item::INT_ITEM: case Item::REAL_ITEM: case Item::DECIMAL_ITEM: - DBUG_RETURN(spider_db_open_item_int(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_int(item, field, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); case Item::CACHE_ITEM: - DBUG_RETURN(spider_db_open_item_cache((Item_cache *)item, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_cache((Item_cache *) item, field, + spider, str, alias, alias_length, + dbton_id, use_fields, fields)); case Item::INSERT_VALUE_ITEM: - DBUG_RETURN(spider_db_open_item_insert_value((Item_insert_value *)item, - spider, str, alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_insert_value((Item_insert_value *) item, + field, spider, str, + alias, alias_length, + dbton_id, + use_fields, fields)); case Item::SUBSELECT_ITEM: case Item::TRIGGER_FIELD_ITEM: #ifdef SPIDER_HAS_EXPR_CACHE_ITEM @@ -8525,6 +8608,7 @@ int spider_db_print_item_type( } break; } + DBUG_RETURN(0); } @@ -8556,8 +8640,9 @@ restart_first: { if (str) restart_pos = str->length(); - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields))) { if ( str && @@ -8590,8 +8675,9 @@ restart_first: str->q_append(SPIDER_SQL_SPACE_STR, SPIDER_SQL_SPACE_LEN); } - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields))) { if ( str && @@ -8827,8 +8913,9 @@ int spider_db_open_item_ref( } DBUG_RETURN(0); } - DBUG_RETURN(spider_db_print_item_type(*(item_ref->ref), spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_print_item_type(*(item_ref->ref), NULL, spider, + str, alias, alias_length, dbton_id, + use_fields, fields)); } DBUG_RETURN(spider_db_open_item_ident((Item_ident *) item_ref, spider, str, alias, alias_length, dbton_id, use_fields, fields)); @@ -8857,8 +8944,9 @@ int spider_db_open_item_row( for (roop_count = 0; roop_count < cols; roop_count++) { item = item_row->element_index(roop_count); - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); if (str) { @@ -8868,8 +8956,9 @@ int spider_db_open_item_row( } } item = item_row->element_index(roop_count); - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); if (str) { @@ -8881,104 +8970,251 @@ int spider_db_open_item_row( DBUG_RETURN(0); } -int spider_db_open_item_string( - Item *item, - ha_spider *spider, - spider_string *str, - const char *alias, - uint alias_length, - uint dbton_id, - bool use_fields, - spider_fields *fields -) { +/** + Print a string value within a generated statement. + + @param item String value to print. + @param field Field related to the string value. + @param spider Spider. + @param str String into which the value is to be printed. + @param alias Name related to the string value. + @param alias_length Length of the name. + @param dbton_id Spider Db/Table id. + @param use_fields Use fields or exchange fields. + @param fields Array of fields in an expression containing + the string value. + + @return Error code. +*/ + +int spider_db_open_item_string(Item *item, Field *field, ha_spider *spider, + spider_string *str, + const char *alias, uint alias_length, + uint dbton_id, + bool use_fields, spider_fields *fields) +{ + int error_num = 0; DBUG_ENTER("spider_db_open_item_string"); + if (str) { + THD *thd = NULL; + TABLE *table; + my_bitmap_map *saved_map; + Time_zone *saved_time_zone; char tmp_buf[MAX_FIELD_WIDTH]; spider_string tmp_str(tmp_buf, MAX_FIELD_WIDTH, str->charset()); String *tmp_str2; + String str_value; tmp_str.init_calc_mem(126); + if (!(tmp_str2 = item->val_str(tmp_str.get_str()))) { if (str->reserve(SPIDER_SQL_NULL_LEN)) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); + { + error_num = HA_ERR_OUT_OF_MEM; + goto error; + } str->q_append(SPIDER_SQL_NULL_STR, SPIDER_SQL_NULL_LEN); - } else { + } + else + { + if (field && field->type() == FIELD_TYPE_TIMESTAMP) + { + /* + Store the string value in the field. This is necessary + when the statement contains more than one value for the + same field. + */ + table = field->table; + thd = table->in_use; + saved_map = dbug_tmp_use_all_columns(table, table->write_set); + item->save_in_field(field, FALSE); + saved_time_zone = thd->variables.time_zone; + thd->variables.time_zone = UTC; + + /* Retrieve the stored value converted to UTC */ + tmp_str2 = field->val_str(&str_value); + + if (!tmp_str2) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error; + } + } + if (str->reserve(SPIDER_SQL_VALUE_QUOTE_LEN * 2 + - tmp_str2->length() * 2)) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - tmp_str.mem_calc(); + tmp_str2->length() * 2)) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error; + } + if (!thd) + tmp_str.mem_calc(); str->q_append(SPIDER_SQL_VALUE_QUOTE_STR, SPIDER_SQL_VALUE_QUOTE_LEN); str->append_escape_string(tmp_str2->ptr(), tmp_str2->length()); - if ( - str->reserve(SPIDER_SQL_VALUE_QUOTE_LEN) - ) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); + if (str->reserve(SPIDER_SQL_VALUE_QUOTE_LEN)) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error; + } str->q_append(SPIDER_SQL_VALUE_QUOTE_STR, SPIDER_SQL_VALUE_QUOTE_LEN); } + +error: + if (thd) + { + thd->variables.time_zone = saved_time_zone; + dbug_tmp_restore_column_map(table->write_set, saved_map); + } } - DBUG_RETURN(0); + + DBUG_RETURN(error_num); } -int spider_db_open_item_int( - Item *item, - ha_spider *spider, - spider_string *str, - const char *alias, - uint alias_length, - uint dbton_id, - bool use_fields, - spider_fields *fields -) { +/** + Print an integer value within a generated statement. + + @param item Integer value to print. + @param field Field related to the integer value. + @param spider Spider. + @param str String into which the value is to be printed. + @param alias Name related to the integer value. + @param alias_length Length of the name. + @param dbton_id Spider Db/Table id. + @param use_fields Use fields or exchange fields. + @param fields Array of fields in an expression containing + the integer value. + + @return Error code. +*/ + +int spider_db_open_item_int(Item *item, Field *field, ha_spider *spider, + spider_string *str, + const char *alias, uint alias_length, + uint dbton_id, + bool use_fields, spider_fields *fields) +{ + int error_num = 0; DBUG_ENTER("spider_db_open_item_int"); + if (str) { + THD *thd = NULL; + TABLE *table; + bool print_quoted_string; + my_bitmap_map *saved_map; + Time_zone *saved_time_zone; char tmp_buf[MAX_FIELD_WIDTH]; spider_string tmp_str(tmp_buf, MAX_FIELD_WIDTH, str->charset()); + String str_value; String *tmp_str2; tmp_str.init_calc_mem(127); + if (!(tmp_str2 = item->val_str(tmp_str.get_str()))) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - tmp_str.mem_calc(); -#ifdef SPIDER_ITEM_HAS_CMP_TYPE - DBUG_PRINT("info",("spider cmp_type=%u", item->cmp_type())); - if (item->cmp_type() == TIME_RESULT) { + error_num = HA_ERR_OUT_OF_MEM; + goto error; + } + tmp_str.mem_calc(); + + if (field && field->type() == FIELD_TYPE_TIMESTAMP) + { + /* + Store the int value in the field. This is necessary + when the statement contains more than one value for the + same field. + */ + table = field->table; + thd = table->in_use; + saved_map = dbug_tmp_use_all_columns(table, table->write_set); + item->save_in_field(field, FALSE); + saved_time_zone = thd->variables.time_zone; + thd->variables.time_zone = UTC; + print_quoted_string = TRUE; + } + else + { +#ifdef SPIDER_ITEM_HAS_CMP_TYPE + DBUG_PRINT("info", ("spider cmp_type=%u", item->cmp_type())); + if (item->cmp_type() == TIME_RESULT) + print_quoted_string = TRUE; + else +#endif + print_quoted_string = FALSE; + } + + if (print_quoted_string) + { + if (thd) + { + /* Retrieve the stored value converted to UTC */ + tmp_str2 = field->val_str(&str_value); + + if (!tmp_str2) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error; + } + } + if (str->reserve(SPIDER_SQL_VALUE_QUOTE_LEN * 2 + tmp_str2->length())) DBUG_RETURN(HA_ERR_OUT_OF_MEM); str->q_append(SPIDER_SQL_VALUE_QUOTE_STR, SPIDER_SQL_VALUE_QUOTE_LEN); str->append(*tmp_str2); str->q_append(SPIDER_SQL_VALUE_QUOTE_STR, SPIDER_SQL_VALUE_QUOTE_LEN); - } else { -#endif - if (str->append(*tmp_str2)) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); -#ifdef SPIDER_ITEM_HAS_CMP_TYPE } -#endif + else + { + if (str->append(*tmp_str2)) + error_num = HA_ERR_OUT_OF_MEM; + } + +error: + if (thd) + { + thd->variables.time_zone = saved_time_zone; + dbug_tmp_restore_column_map(table->write_set, saved_map); + } } - DBUG_RETURN(0); + + DBUG_RETURN(error_num); } -int spider_db_open_item_cache( - Item_cache *item_cache, - ha_spider *spider, - spider_string *str, - const char *alias, - uint alias_length, - uint dbton_id, - bool use_fields, - spider_fields *fields -) { +/** + Print a cached value within a generated statement. + + @param item Cached value to print. + @param field Field related to the cached value. + @param spider Spider. + @param str String into which the value is to be printed. + @param alias Name related to the cached value. + @param alias_length Length of the name. + @param dbton_id Spider Db/Table id. + @param use_fields Use fields or exchange fields. + @param fields Array of fields in the expression containing + the cached value. + + @return Error code. +*/ + +int spider_db_open_item_cache(Item_cache *item_cache, Field *field, + ha_spider *spider, spider_string *str, + const char *alias, uint alias_length, + uint dbton_id, + bool use_fields, spider_fields *fields) +{ DBUG_ENTER("spider_db_open_item_cache"); if (!item_cache->const_item()) DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); DBUG_PRINT("info",("spider result_type=%u", item_cache->result_type())); + switch (item_cache->result_type()) { case STRING_RESULT: - DBUG_RETURN(spider_db_open_item_string(item_cache, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_string(item_cache, field, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); case ROW_RESULT: { int error_num; @@ -8993,9 +9229,11 @@ int spider_db_open_item_cache( for (roop_count = 0; roop_count < item_count; ++roop_count) { if ((error_num = spider_db_open_item_cache( - (Item_cache *) item_cache_row->element_index(roop_count), - spider, str, alias, alias_length, dbton_id, use_fields, fields - ))) { + (Item_cache *) + item_cache_row->element_index(roop_count), + NULL, spider, str, alias, alias_length, + dbton_id, use_fields, fields))) + { DBUG_RETURN(error_num); } if (str) @@ -9006,9 +9244,11 @@ int spider_db_open_item_cache( } } if ((error_num = spider_db_open_item_cache( - (Item_cache *) item_cache_row->element_index(roop_count), - spider, str, alias, alias_length, dbton_id, use_fields, fields - ))) { + (Item_cache *) + item_cache_row->element_index(roop_count), + NULL, spider, str, alias, alias_length, + dbton_id, use_fields, fields))) + { DBUG_RETURN(error_num); } if (str) @@ -9026,22 +9266,38 @@ int spider_db_open_item_cache( default: break; } - DBUG_RETURN(spider_db_open_item_int(item_cache, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + + DBUG_RETURN(spider_db_open_item_int(item_cache, field, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); } -int spider_db_open_item_insert_value( - Item_insert_value *item_insert_value, - ha_spider *spider, - spider_string *str, - const char *alias, - uint alias_length, - uint dbton_id, - bool use_fields, - spider_fields *fields -) { +/** + Print an INSERT value within a generated INSERT statement. + + @param item INSERT value to print. + @param field Field related to the INSERT value. + @param spider Spider. + @param str String into which the value is to be printed. + @param alias Name related to the INSERT value. + @param alias_length Length of the name. + @param dbton_id Spider Db/Table id. + @param use_fields Use fields or exchange fields. + @param fields Array of fields in the expression. + + @return Error code. +*/ + +int spider_db_open_item_insert_value(Item_insert_value *item_insert_value, + Field *field, ha_spider *spider, + spider_string *str, + const char *alias, uint alias_length, + uint dbton_id, + bool use_fields, spider_fields *fields) +{ int error_num; DBUG_ENTER("spider_db_open_item_insert_value"); + if (item_insert_value->arg) { if (str) @@ -9051,8 +9307,10 @@ int spider_db_open_item_insert_value( str->q_append(SPIDER_SQL_VALUES_STR, SPIDER_SQL_VALUES_LEN); str->q_append(SPIDER_SQL_OPEN_PAREN_STR, SPIDER_SQL_OPEN_PAREN_LEN); } - if ((error_num = spider_db_print_item_type(item_insert_value->arg, spider, - str, alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item_insert_value->arg, field, + spider, str, alias, + alias_length, dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); if (str) { @@ -9061,6 +9319,7 @@ int spider_db_open_item_insert_value( str->q_append(SPIDER_SQL_CLOSE_PAREN_STR, SPIDER_SQL_CLOSE_PAREN_LEN); } } + DBUG_RETURN(0); } @@ -9116,9 +9375,10 @@ int spider_db_append_update_columns( while ((field = fi++)) { value = vi++; - if ((error_num = spider_db_print_item_type( - (Item *) field, spider, str, alias, alias_length, dbton_id, - use_fields, fields))) + if ((error_num = spider_db_print_item_type((Item *) field, NULL, spider, + str, alias, alias_length, + dbton_id, + use_fields, fields))) { if ( error_num == ER_SPIDER_COND_SKIP_NUM && @@ -9136,9 +9396,12 @@ int spider_db_append_update_columns( DBUG_RETURN(HA_ERR_OUT_OF_MEM); str->q_append(SPIDER_SQL_EQUAL_STR, SPIDER_SQL_EQUAL_LEN); } - if ((error_num = spider_db_print_item_type( - (Item *) value, spider, str, alias, alias_length, dbton_id, - use_fields, fields))) + if ((error_num = spider_db_print_item_type((Item *) value, + ((Item_field *) field)->field, + spider, str, + alias, alias_length, + dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); if (str) { diff --git a/storage/spider/spd_db_conn.h b/storage/spider/spd_db_conn.h index 7977e61da58..879ab3540c1 100644 --- a/storage/spider/spd_db_conn.h +++ b/storage/spider/spd_db_conn.h @@ -838,8 +838,18 @@ int spider_db_flush_logs( ha_spider *spider ); +Field *spider_db_find_field_in_item_list( + Item **item_list, + uint item_count, + uint start_item, + spider_string *str, + const char *func_name, + int func_name_length +); + int spider_db_print_item_type( Item *item, + Field *field, ha_spider *spider, spider_string *str, const char *alias, @@ -930,6 +940,7 @@ int spider_db_open_item_row( int spider_db_open_item_string( Item *item, + Field *field, ha_spider *spider, spider_string *str, const char *alias, @@ -941,6 +952,7 @@ int spider_db_open_item_string( int spider_db_open_item_int( Item *item, + Field *field, ha_spider *spider, spider_string *str, const char *alias, @@ -952,6 +964,7 @@ int spider_db_open_item_int( int spider_db_open_item_cache( Item_cache *item_cache, + Field *field, ha_spider *spider, spider_string *str, const char *alias, @@ -963,6 +976,7 @@ int spider_db_open_item_cache( int spider_db_open_item_insert_value( Item_insert_value *item_insert_value, + Field *field, ha_spider *spider, spider_string *str, const char *alias, diff --git a/storage/spider/spd_db_include.h b/storage/spider/spd_db_include.h index 2913f911587..639bb6dac2d 100644 --- a/storage/spider/spd_db_include.h +++ b/storage/spider/spd_db_include.h @@ -17,6 +17,7 @@ #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) #include "hstcpcli.hpp" #endif +#include "tztime.h" #define SPIDER_DBTON_SIZE 15 diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index 21bbaaea2c9..db685c50b85 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -179,6 +179,9 @@ static const char *spider_db_timefunc_interval_str[] = " minute_microsecond", " second_microsecond" }; +/* UTC time zone for timestamp columns */ +Time_zone *UTC = 0; + int spider_mysql_init() { DBUG_ENTER("spider_mysql_init"); @@ -3192,9 +3195,13 @@ int spider_db_mysql_util::append_column_value( spider_string tmp_str(buf, MAX_FIELD_WIDTH, &my_charset_bin); String *ptr; uint length; + THD *thd = field->table->in_use; + Time_zone *saved_time_zone = thd->variables.time_zone; DBUG_ENTER("spider_db_mysql_util::append_column_value"); tmp_str.init_calc_mem(113); + thd->variables.time_zone = UTC; + if (new_ptr) { if ( @@ -3205,7 +3212,8 @@ int spider_db_mysql_util::append_column_value( tmp_str.set_quick((char *) new_ptr + HA_KEY_BLOB_LENGTH, length, &my_charset_bin); ptr = tmp_str.get_str(); - } else if (field->type() == MYSQL_TYPE_GEOMETRY) + } + else if (field->type() == MYSQL_TYPE_GEOMETRY) { /* uint mlength = SIZEOF_STORED_DOUBLE, lcnt; @@ -3282,14 +3290,21 @@ int spider_db_mysql_util::append_column_value( tmp_str.q_append((char *) new_ptr + SIZEOF_STORED_DOUBLE * 3, SIZEOF_STORED_DOUBLE); ptr = tmp_str.get_str(); - } else { + } + else + { ptr = field->val_str(tmp_str.get_str(), new_ptr); tmp_str.mem_calc(); } - } else { + } + else + { ptr = field->val_str(tmp_str.get_str()); tmp_str.mem_calc(); } + + thd->variables.time_zone = saved_time_zone; + DBUG_PRINT("info", ("spider field->type() is %d", field->type())); DBUG_PRINT("info", ("spider ptr->length() is %d", ptr->length())); /* @@ -3335,7 +3350,8 @@ int spider_db_mysql_util::append_column_value( append_escaped_util(str, tmp_str2.get_str()) ) DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } else if (str->append(*ptr)) + } + else if (str->append(*ptr)) DBUG_RETURN(HA_ERR_OUT_OF_MEM); if (str->reserve(SPIDER_SQL_VALUE_QUOTE_LEN)) DBUG_RETURN(HA_ERR_OUT_OF_MEM); @@ -3604,12 +3620,13 @@ int spider_db_mysql_util::open_item_func( uint dbton_id = spider_dbton_mysql.dbton_id; int error_num; Item *item, **item_list = item_func->arguments(); - uint roop_count, item_count = item_func->argument_count(), start_item = 0; + Field *field; + uint loop_count, item_count = item_func->argument_count(), start_item = 0; const char *func_name = SPIDER_SQL_NULL_CHAR_STR, - *separete_str = SPIDER_SQL_NULL_CHAR_STR, + *separator_str = SPIDER_SQL_NULL_CHAR_STR, *last_str = SPIDER_SQL_NULL_CHAR_STR; int func_name_length = SPIDER_SQL_NULL_CHAR_LEN, - separete_str_length = SPIDER_SQL_NULL_CHAR_LEN, + separator_str_length = SPIDER_SQL_NULL_CHAR_LEN, last_str_length = SPIDER_SQL_NULL_CHAR_LEN; int use_pushdown_udf; bool merge_func = FALSE; @@ -3675,8 +3692,9 @@ int spider_db_mysql_util::open_item_func( ) { if (str) str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); - DBUG_RETURN(spider_db_open_item_int(item_func, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_int(item_func, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); } else if ( !strncasecmp("case", func_name, func_name_length) ) { @@ -3695,8 +3713,8 @@ int spider_db_mysql_util::open_item_func( alias, alias_length, dbton_id, use_fields, fields))) DBUG_RETURN(error_num); } - for (roop_count = 0; roop_count < item_func_case->ncases; - roop_count += 2) + for (loop_count = 0; loop_count < item_func_case->ncases; + loop_count += 2) { if (str) { @@ -3705,7 +3723,7 @@ int spider_db_mysql_util::open_item_func( str->q_append(SPIDER_SQL_WHEN_STR, SPIDER_SQL_WHEN_LEN); } if ((error_num = spider_db_print_item_type( - item_list[roop_count], spider, str, + item_list[loop_count], spider, str, alias, alias_length, dbton_id, use_fields, fields))) DBUG_RETURN(error_num); if (str) @@ -3715,7 +3733,7 @@ int spider_db_mysql_util::open_item_func( str->q_append(SPIDER_SQL_THEN_STR, SPIDER_SQL_THEN_LEN); } if ((error_num = spider_db_print_item_type( - item_list[roop_count + 1], spider, str, + item_list[loop_count + 1], spider, str, alias, alias_length, dbton_id, use_fields, fields))) DBUG_RETURN(error_num); } @@ -3765,8 +3783,10 @@ int spider_db_mysql_util::open_item_func( ) { if (str) str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); - DBUG_RETURN(spider_db_open_item_string(item_func, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_string(item_func, NULL, spider, str, + alias, alias_length, + dbton_id, + use_fields, fields)); } else if ( !strncasecmp("convert", func_name, func_name_length) ) { @@ -3790,8 +3810,9 @@ int spider_db_mysql_util::open_item_func( ) { if (str) str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); - DBUG_RETURN(spider_db_open_item_string(item_func, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_string(item_func, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); } else if (func_name_length == 9 && !strncasecmp("isnottrue", func_name, func_name_length) ) { @@ -3817,8 +3838,8 @@ int spider_db_mysql_util::open_item_func( } func_name = SPIDER_SQL_COMMA_STR; func_name_length = SPIDER_SQL_COMMA_LEN; - separete_str = SPIDER_SQL_COMMA_STR; - separete_str_length = SPIDER_SQL_COMMA_LEN; + separator_str = SPIDER_SQL_COMMA_STR; + separator_str_length = SPIDER_SQL_COMMA_LEN; break; } } else if (func_name_length == 12) @@ -3906,8 +3927,10 @@ int spider_db_mysql_util::open_item_func( { if (str) str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); - DBUG_RETURN(spider_db_open_item_string(item_func, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_string(item_func, NULL, spider, str, + alias, alias_length, + dbton_id, + use_fields, fields)); } else if (!strncasecmp("timestampdiff", func_name, func_name_length)) { #ifdef ITEM_FUNC_TIMESTAMPDIFF_ARE_PUBLIC @@ -4232,8 +4255,11 @@ int spider_db_mysql_util::open_item_func( func_name = spider_db_timefunc_interval_str[ item_date_add_interval->int_type]; func_name_length = strlen(func_name); - if ((error_num = spider_db_print_item_type(item_list[0], spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item_list[0], NULL, + spider, str, + alias, alias_length, + dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); if (str) { @@ -4249,8 +4275,11 @@ int spider_db_mysql_util::open_item_func( str->q_append(SPIDER_SQL_INTERVAL_STR, SPIDER_SQL_INTERVAL_LEN); } } - if ((error_num = spider_db_print_item_type(item_list[1], spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item_list[1], NULL, + spider, str, + alias, alias_length, + dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); if (str) { @@ -4272,16 +4301,17 @@ int spider_db_mysql_util::open_item_func( } func_name = SPIDER_SQL_COMMA_STR; func_name_length = SPIDER_SQL_COMMA_LEN; - separete_str = SPIDER_SQL_COMMA_STR; - separete_str_length = SPIDER_SQL_COMMA_LEN; + separator_str = SPIDER_SQL_COMMA_STR; + separator_str_length = SPIDER_SQL_COMMA_LEN; last_str = SPIDER_SQL_CLOSE_PAREN_STR; last_str_length = SPIDER_SQL_CLOSE_PAREN_LEN; break; case Item_func::NOW_FUNC: if (str) str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); - DBUG_RETURN(spider_db_open_item_string(item_func, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_string(item_func, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); case Item_func::CHAR_TYPECAST_FUNC: DBUG_PRINT("info",("spider CHAR_TYPECAST_FUNC")); { @@ -4406,15 +4436,15 @@ int spider_db_mysql_util::open_item_func( { func_name = SPIDER_SQL_NOT_IN_STR; func_name_length = SPIDER_SQL_NOT_IN_LEN; - separete_str = SPIDER_SQL_COMMA_STR; - separete_str_length = SPIDER_SQL_COMMA_LEN; + separator_str = SPIDER_SQL_COMMA_STR; + separator_str_length = SPIDER_SQL_COMMA_LEN; last_str = SPIDER_SQL_CLOSE_PAREN_STR; last_str_length = SPIDER_SQL_CLOSE_PAREN_LEN; } else { func_name = SPIDER_SQL_IN_STR; func_name_length = SPIDER_SQL_IN_LEN; - separete_str = SPIDER_SQL_COMMA_STR; - separete_str_length = SPIDER_SQL_COMMA_LEN; + separator_str = SPIDER_SQL_COMMA_STR; + separator_str_length = SPIDER_SQL_COMMA_LEN; last_str = SPIDER_SQL_CLOSE_PAREN_STR; last_str_length = SPIDER_SQL_CLOSE_PAREN_LEN; } @@ -4424,13 +4454,13 @@ int spider_db_mysql_util::open_item_func( { func_name = SPIDER_SQL_NOT_BETWEEN_STR; func_name_length = SPIDER_SQL_NOT_BETWEEN_LEN; - separete_str = SPIDER_SQL_AND_STR; - separete_str_length = SPIDER_SQL_AND_LEN; + separator_str = SPIDER_SQL_AND_STR; + separator_str_length = SPIDER_SQL_AND_LEN; } else { func_name = (char*) item_func->func_name(); func_name_length = strlen(func_name); - separete_str = SPIDER_SQL_AND_STR; - separete_str_length = SPIDER_SQL_AND_LEN; + separator_str = SPIDER_SQL_AND_STR; + separator_str_length = SPIDER_SQL_AND_LEN; } break; case Item_func::UDF_FUNC: @@ -4451,8 +4481,8 @@ int spider_db_mysql_util::open_item_func( } func_name = SPIDER_SQL_COMMA_STR; func_name_length = SPIDER_SQL_COMMA_LEN; - separete_str = SPIDER_SQL_COMMA_STR; - separete_str_length = SPIDER_SQL_COMMA_LEN; + separator_str = SPIDER_SQL_COMMA_STR; + separator_str_length = SPIDER_SQL_COMMA_LEN; last_str = SPIDER_SQL_CLOSE_PAREN_STR; last_str_length = SPIDER_SQL_CLOSE_PAREN_LEN; break; @@ -4472,11 +4502,13 @@ int spider_db_mysql_util::open_item_func( if (str) str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); if (item_func->result_type() == STRING_RESULT) - DBUG_RETURN(spider_db_open_item_string(item_func, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_string(item_func, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); else - DBUG_RETURN(spider_db_open_item_int(item_func, spider, str, - alias, alias_length, dbton_id, use_fields, fields)); + DBUG_RETURN(spider_db_open_item_int(item_func, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields)); case Item_func::FT_FUNC: if (spider_db_check_ft_idx(item_func, spider) == MAX_KEY) DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); @@ -4487,8 +4519,8 @@ int spider_db_mysql_util::open_item_func( DBUG_RETURN(HA_ERR_OUT_OF_MEM); str->q_append(SPIDER_SQL_MATCH_STR, SPIDER_SQL_MATCH_LEN); } - separete_str = SPIDER_SQL_COMMA_STR; - separete_str_length = SPIDER_SQL_COMMA_LEN; + separator_str = SPIDER_SQL_COMMA_STR; + separator_str_length = SPIDER_SQL_COMMA_LEN; last_str = SPIDER_SQL_CLOSE_PAREN_STR; last_str_length = SPIDER_SQL_CLOSE_PAREN_LEN; break; @@ -4505,8 +4537,8 @@ int spider_db_mysql_util::open_item_func( } func_name = SPIDER_SQL_COMMA_STR; func_name_length = SPIDER_SQL_COMMA_LEN; - separete_str = SPIDER_SQL_COMMA_STR; - separete_str_length = SPIDER_SQL_COMMA_LEN; + separator_str = SPIDER_SQL_COMMA_STR; + separator_str_length = SPIDER_SQL_COMMA_LEN; last_str = SPIDER_SQL_CLOSE_PAREN_STR; last_str_length = SPIDER_SQL_CLOSE_PAREN_LEN; break; @@ -4537,8 +4569,8 @@ int spider_db_mysql_util::open_item_func( } func_name = SPIDER_SQL_COMMA_STR; func_name_length = SPIDER_SQL_COMMA_LEN; - separete_str = SPIDER_SQL_COMMA_STR; - separete_str_length = SPIDER_SQL_COMMA_LEN; + separator_str = SPIDER_SQL_COMMA_STR; + separator_str_length = SPIDER_SQL_COMMA_LEN; last_str = SPIDER_SQL_CLOSE_PAREN_STR; last_str_length = SPIDER_SQL_CLOSE_PAREN_LEN; break; @@ -4571,23 +4603,39 @@ int spider_db_mysql_util::open_item_func( } DBUG_PRINT("info",("spider func_name = %s", func_name)); DBUG_PRINT("info",("spider func_name_length = %d", func_name_length)); - DBUG_PRINT("info",("spider separete_str = %s", separete_str)); - DBUG_PRINT("info",("spider separete_str_length = %d", separete_str_length)); + DBUG_PRINT("info",("spider separator_str = %s", separator_str)); + DBUG_PRINT("info",("spider separator_str_length = %d", separator_str_length)); DBUG_PRINT("info",("spider last_str = %s", last_str)); DBUG_PRINT("info",("spider last_str_length = %d", last_str_length)); + if (item_count) { + /* Find the field in the list of items of the expression tree */ + field = spider_db_find_field_in_item_list(item_list, + item_count, start_item, + str, + func_name, func_name_length); + item_count--; - for (roop_count = start_item; roop_count < item_count; roop_count++) + + /* + Loop through the items of the current function expression to + print its portion of the statement + */ + for (loop_count = start_item; loop_count < item_count; loop_count++) { - item = item_list[roop_count]; - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + item = item_list[loop_count]; + if ((error_num = spider_db_print_item_type(item, field, spider, str, + alias, alias_length, + dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); - if (roop_count == 1) + + if (loop_count == 1) { - func_name = separete_str; - func_name_length = separete_str_length; + /* Remaining operands need to be preceded by the separator */ + func_name = separator_str; + func_name_length = separator_str_length; } if (str) { @@ -4598,11 +4646,16 @@ int spider_db_mysql_util::open_item_func( str->q_append(SPIDER_SQL_SPACE_STR, SPIDER_SQL_SPACE_LEN); } } - item = item_list[roop_count]; - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + + /* Print the last operand value */ + item = item_list[loop_count]; + if ((error_num = spider_db_print_item_type(item, field, spider, str, + alias, alias_length, + dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); } + if (item_func->functype() == Item_func::FT_FUNC) { Item_func_match *item_func_match = (Item_func_match *)item_func; @@ -4613,8 +4666,9 @@ int spider_db_mysql_util::open_item_func( str->q_append(SPIDER_SQL_AGAINST_STR, SPIDER_SQL_AGAINST_LEN); } item = item_list[0]; - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); if (str) { @@ -4701,8 +4755,11 @@ int spider_db_mysql_util::open_item_sum_func( for (roop_count = 0; roop_count < item_count; roop_count++) { item = args[roop_count]; - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item, NULL, + spider, str, + alias, alias_length, + dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); if (str) { @@ -4712,8 +4769,10 @@ int spider_db_mysql_util::open_item_sum_func( } } item = args[roop_count]; - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item, NULL, spider, str, + alias, alias_length, + dbton_id, + use_fields, fields))) DBUG_RETURN(error_num); } if (str) @@ -7592,8 +7651,9 @@ int spider_mysql_handler::check_item_type( int error_num; DBUG_ENTER("spider_mysql_handler::check_item_type"); DBUG_PRINT("info",("spider this=%p", this)); - error_num = spider_db_print_item_type(item, spider, NULL, NULL, 0, - spider_dbton_mysql.dbton_id, FALSE, NULL); + error_num = spider_db_print_item_type(item, NULL, spider, NULL, NULL, 0, + spider_dbton_mysql.dbton_id, + FALSE, NULL); DBUG_RETURN(error_num); } @@ -8391,9 +8451,11 @@ int spider_mysql_handler::append_condition( str->q_append(SPIDER_SQL_AND_STR, SPIDER_SQL_AND_LEN); } } - if ((error_num = spider_db_print_item_type( - (Item *) tmp_cond->cond, spider, str, alias, alias_length, - spider_dbton_mysql.dbton_id, FALSE, NULL))) + if ((error_num = spider_db_print_item_type((Item *) tmp_cond->cond, + NULL, spider, str, + alias, alias_length, + spider_dbton_mysql.dbton_id, + FALSE, NULL))) { if (str && error_num == ER_SPIDER_COND_SKIP_NUM) { @@ -8715,8 +8777,10 @@ int spider_mysql_handler::append_group_by( str->q_append(SPIDER_SQL_GROUP_STR, SPIDER_SQL_GROUP_LEN); for (; group; group = group->next) { - if ((error_num = spider_db_print_item_type((*group->item), spider, str, - alias, alias_length, spider_dbton_mysql.dbton_id, FALSE, NULL))) + if ((error_num = spider_db_print_item_type((*group->item), NULL, spider, + str, alias, alias_length, + spider_dbton_mysql.dbton_id, + FALSE, NULL))) { DBUG_RETURN(error_num); } @@ -8908,8 +8972,9 @@ int spider_mysql_handler::append_key_order_for_direct_order_limit_with_alias( order = order->next) { if ((error_num = - spider_db_print_item_type((*order->item), spider, str, alias, - alias_length, spider_dbton_mysql.dbton_id, FALSE, NULL))) + spider_db_print_item_type((*order->item), NULL, spider, str, alias, + alias_length, spider_dbton_mysql.dbton_id, + FALSE, NULL))) { DBUG_PRINT("info",("spider error=%d", error_num)); DBUG_RETURN(error_num); @@ -12768,8 +12833,10 @@ int spider_mysql_handler::append_item_type_part( default: DBUG_RETURN(0); } - error_num = spider_db_print_item_type(item, spider, str, alias, alias_length, - spider_dbton_mysql.dbton_id, use_fields, fields); + error_num = spider_db_print_item_type(item, NULL, spider, str, + alias, alias_length, + spider_dbton_mysql.dbton_id, + use_fields, fields); DBUG_RETURN(error_num); } @@ -12815,8 +12882,9 @@ int spider_mysql_handler::append_list_item_select( DBUG_PRINT("info",("spider this=%p", this)); while ((item = it++)) { - if ((error_num = spider_db_print_item_type(item, spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type(item, NULL, spider, str, + alias, alias_length, dbton_id, + use_fields, fields))) { DBUG_RETURN(error_num); } @@ -12883,8 +12951,10 @@ int spider_mysql_handler::append_group_by( str->q_append(SPIDER_SQL_GROUP_STR, SPIDER_SQL_GROUP_LEN); for (; order; order = order->next) { - if ((error_num = spider_db_print_item_type((*order->item), spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type((*order->item), NULL, spider, + str, alias, alias_length, + dbton_id, + use_fields, fields))) { DBUG_RETURN(error_num); } @@ -12941,8 +13011,10 @@ int spider_mysql_handler::append_order_by( str->q_append(SPIDER_SQL_ORDER_STR, SPIDER_SQL_ORDER_LEN); for (; order; order = order->next) { - if ((error_num = spider_db_print_item_type((*order->item), spider, str, - alias, alias_length, dbton_id, use_fields, fields))) + if ((error_num = spider_db_print_item_type((*order->item), NULL, spider, + str, alias, alias_length, + dbton_id, + use_fields, fields))) { DBUG_RETURN(error_num); } diff --git a/storage/spider/spd_db_mysql.h b/storage/spider/spd_db_mysql.h index 766c15971ec..25cad01c66c 100644 --- a/storage/spider/spd_db_mysql.h +++ b/storage/spider/spd_db_mysql.h @@ -13,6 +13,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include "tztime.h" + class spider_db_mysql_util: public spider_db_util { public: diff --git a/storage/spider/spd_db_oracle.h b/storage/spider/spd_db_oracle.h index 2e9da5cab84..6962ff4884f 100644 --- a/storage/spider/spd_db_oracle.h +++ b/storage/spider/spd_db_oracle.h @@ -13,6 +13,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include "tztime.h" + class spider_db_oracle; class spider_db_oracle_result; diff --git a/storage/spider/spd_group_by_handler.cc b/storage/spider/spd_group_by_handler.cc index 673248d0530..a5328464bb6 100644 --- a/storage/spider/spd_group_by_handler.cc +++ b/storage/spider/spd_group_by_handler.cc @@ -1727,8 +1727,8 @@ group_by_handler *spider_create_group_by_handler( while ((item = it++)) { DBUG_PRINT("info",("spider select item=%p", item)); - if (spider_db_print_item_type(item, spider, NULL, NULL, 0, - roop_count, TRUE, fields_arg)) + if (spider_db_print_item_type(item, NULL, spider, NULL, NULL, 0, + roop_count, TRUE, fields_arg)) { DBUG_PRINT("info",("spider dbton_id=%d can't create select", roop_count)); spider_clear_bit(dbton_bitmap, roop_count); @@ -1741,8 +1741,9 @@ group_by_handler *spider_create_group_by_handler( DBUG_PRINT("info",("spider query->where=%p", query->where)); if (query->where) { - if (spider_db_print_item_type(query->where, spider, NULL, NULL, 0, - roop_count, TRUE, fields_arg)) + if (spider_db_print_item_type(query->where, NULL, spider, NULL, + NULL, 0, roop_count, + TRUE, fields_arg)) { DBUG_PRINT("info",("spider dbton_id=%d can't create where", roop_count)); spider_clear_bit(dbton_bitmap, roop_count); @@ -1757,8 +1758,9 @@ group_by_handler *spider_create_group_by_handler( { for (order = query->group_by; order; order = order->next) { - if (spider_db_print_item_type((*order->item), spider, NULL, NULL, 0, - roop_count, TRUE, fields_arg)) + if (spider_db_print_item_type((*order->item), NULL, spider, NULL, + NULL, 0, roop_count, + TRUE, fields_arg)) { DBUG_PRINT("info",("spider dbton_id=%d can't create group by", roop_count)); spider_clear_bit(dbton_bitmap, roop_count); @@ -1775,8 +1777,9 @@ group_by_handler *spider_create_group_by_handler( { for (order = query->order_by; order; order = order->next) { - if (spider_db_print_item_type((*order->item), spider, NULL, NULL, 0, - roop_count, TRUE, fields_arg)) + if (spider_db_print_item_type((*order->item), NULL, spider, NULL, + NULL, 0, roop_count, + TRUE, fields_arg)) { DBUG_PRINT("info",("spider dbton_id=%d can't create order by", roop_count)); spider_clear_bit(dbton_bitmap, roop_count); @@ -1791,8 +1794,9 @@ group_by_handler *spider_create_group_by_handler( DBUG_PRINT("info",("spider query->having=%p", query->having)); if (query->having) { - if (spider_db_print_item_type(query->having, spider, NULL, NULL, 0, - roop_count, TRUE, fields_arg)) + if (spider_db_print_item_type(query->having, NULL, spider, NULL, + NULL, 0, roop_count, + TRUE, fields_arg)) { DBUG_PRINT("info",("spider dbton_id=%d can't create having", roop_count)); spider_clear_bit(dbton_bitmap, roop_count); diff --git a/storage/spider/spd_include.h b/storage/spider/spd_include.h index bede60412e2..79e030b0872 100644 --- a/storage/spider/spd_include.h +++ b/storage/spider/spd_include.h @@ -13,6 +13,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include "tztime.h" + #define SPIDER_DETAIL_VERSION "3.3.13" #define SPIDER_HEX_VERSION 0x0303 diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc index 9a771fe8357..9c7aa4dcdcb 100644 --- a/storage/spider/spd_param.cc +++ b/storage/spider/spd_param.cc @@ -928,26 +928,6 @@ bool spider_param_sync_autocommit( DBUG_RETURN(THDVAR(thd, sync_autocommit)); } -/* - FALSE: no sync - TRUE: sync - */ -static MYSQL_THDVAR_BOOL( - sync_time_zone, /* name */ - PLUGIN_VAR_OPCMDARG, /* opt */ - "Sync time_zone", /* comment */ - NULL, /* check */ - NULL, /* update */ - FALSE /* def */ -); - -bool spider_param_sync_time_zone( - THD *thd -) { - DBUG_ENTER("spider_param_sync_time_zone"); - DBUG_RETURN(THDVAR(thd, sync_time_zone)); -} - /* FALSE: not use TRUE: use @@ -3332,7 +3312,6 @@ static struct st_mysql_sys_var* spider_system_variables[] = { MYSQL_SYSVAR(block_size), MYSQL_SYSVAR(selupd_lock_mode), MYSQL_SYSVAR(sync_autocommit), - MYSQL_SYSVAR(sync_time_zone), MYSQL_SYSVAR(use_default_database), MYSQL_SYSVAR(internal_sql_log_off), MYSQL_SYSVAR(bulk_size), diff --git a/storage/spider/spd_param.h b/storage/spider/spd_param.h index 9a358c54be5..06df06a3129 100644 --- a/storage/spider/spd_param.h +++ b/storage/spider/spd_param.h @@ -107,9 +107,6 @@ int spider_param_selupd_lock_mode( bool spider_param_sync_autocommit( THD *thd ); -bool spider_param_sync_time_zone( - THD *thd -); bool spider_param_use_default_database( THD *thd ); diff --git a/storage/spider/spd_trx.cc b/storage/spider/spd_trx.cc index cf60a0376bb..e09c7fa6c45 100644 --- a/storage/spider/spd_trx.cc +++ b/storage/spider/spd_trx.cc @@ -26,6 +26,7 @@ #include "sql_class.h" #include "sql_partition.h" #include "records.h" +#include "tztime.h" #endif #include "spd_err.h" #include "spd_param.h" @@ -1840,7 +1841,6 @@ int spider_internal_start_trx( SPIDER_TRX *trx = spider->trx; THD *thd = trx->thd; bool sync_autocommit = spider_param_sync_autocommit(thd); - bool sync_time_zone = spider_param_sync_time_zone(thd); double ping_interval_at_trx_start = spider_param_ping_interval_at_trx_start(thd); bool xa_lock = FALSE; @@ -1867,9 +1867,6 @@ int spider_internal_start_trx( if ( (error_num = spider_check_and_set_sql_log_off(thd, conn, &spider->need_mons[link_idx])) || - (sync_time_zone && - (error_num = spider_check_and_set_time_zone(thd, conn, - &spider->need_mons[link_idx]))) || (sync_autocommit && (error_num = spider_check_and_set_autocommit(thd, conn, &spider->need_mons[link_idx]))) From 141a5b24843c626f545a0c2f49013b54da194c0d Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 14 Jul 2018 20:05:46 +0300 Subject: [PATCH 16/29] rpl_row_001.test failed in internal check Problem was as part of SET PASSWORD FOR ROOT, mysql.user table changed compared to how it was originally created. (plugin changed value) --- mysql-test/suite/rpl/include/rpl_row_001.test | 13 +++++++++++++ mysql-test/suite/rpl/r/rpl_row_001.result | 3 +++ 2 files changed, 16 insertions(+) diff --git a/mysql-test/suite/rpl/include/rpl_row_001.test b/mysql-test/suite/rpl/include/rpl_row_001.test index 8eb684e0dff..97b3a93b4ed 100644 --- a/mysql-test/suite/rpl/include/rpl_row_001.test +++ b/mysql-test/suite/rpl/include/rpl_row_001.test @@ -6,9 +6,16 @@ eval LOAD DATA INFILE '$LOAD_FILE' INTO TABLE t1; eval LOAD DATA INFILE '$LOAD_FILE' INTO TABLE t1; SELECT * FROM t1 ORDER BY word LIMIT 10; +# +# Save password row for root +# + +create temporary table tmp select * from mysql.user where host="localhost" and user="root"; + # # Test slave with wrong password # + save_master_pos; connection slave; sync_with_master; @@ -24,6 +31,7 @@ connection master; real_sleep 2; SET PASSWORD FOR root@"localhost" = PASSWORD(''); # Give slave time to connect (will retry every second) + sleep 2; CREATE TABLE t3(n INT); @@ -80,4 +88,9 @@ SELECT n FROM t1; connection master; DROP TABLE t1; + +# resttore old passwords +replace into mysql.user select * from tmp; +drop temporary table tmp; + sync_slave_with_master; diff --git a/mysql-test/suite/rpl/r/rpl_row_001.result b/mysql-test/suite/rpl/r/rpl_row_001.result index 94985548f59..b5b2b370e61 100644 --- a/mysql-test/suite/rpl/r/rpl_row_001.result +++ b/mysql-test/suite/rpl/r/rpl_row_001.result @@ -15,6 +15,7 @@ Aaron Aaron Ababa Ababa +create temporary table tmp select * from mysql.user where host="localhost" and user="root"; connection slave; STOP SLAVE; connection master; @@ -65,5 +66,7 @@ n 3456 connection master; DROP TABLE t1; +replace into mysql.user select * from tmp; +drop temporary table tmp; connection slave; include/rpl_end.inc From 9827c5e1031ca5ebe1c6c7d4f5eac3cce5749d8e Mon Sep 17 00:00:00 2001 From: Sachin Date: Tue, 26 Jun 2018 11:33:58 +0530 Subject: [PATCH 17/29] MDEV-16192 Table 't' is specified twice, both as a target for 'CREATE' and... as a separate source for data Actually MDEV-15867 and MDEV-16192 are same, Slave adds "or replace" to create table stmt. So create table t1 is create or replace on slave. So this bug is not because of replication, We can get this bug on general server if we manually add or replace to create query. Problem:- So if we try to create table t1 (same name as of temp table t1 ) via CREATE or replace TABLE t AS SELECT * FROM t; Since in this query we are creating table from select * from t1 , we call unique_table function to see whether if source and destination table are same. But there is one issue unique_table does not account if source table is tmp table in this case source and destination table can be same. Solution:- We will change find_dup_table to not to look for temp table if CHECK_DUP_SKIP_TEMP_TABLE flag is on. --- mysql-test/r/create_replace_tmp.result | 4 ++++ mysql-test/suite/rpl/r/rpl_15867.result | 9 +++++++++ mysql-test/suite/rpl/t/rpl_15867.test | 11 +++++++++++ mysql-test/t/create_replace_tmp.test | 4 ++++ sql/sql_base.cc | 6 ++++++ sql/sql_base.h | 1 + sql/sql_parse.cc | 4 ++-- 7 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 mysql-test/r/create_replace_tmp.result create mode 100644 mysql-test/suite/rpl/r/rpl_15867.result create mode 100644 mysql-test/suite/rpl/t/rpl_15867.test create mode 100644 mysql-test/t/create_replace_tmp.test diff --git a/mysql-test/r/create_replace_tmp.result b/mysql-test/r/create_replace_tmp.result new file mode 100644 index 00000000000..0239f4d4817 --- /dev/null +++ b/mysql-test/r/create_replace_tmp.result @@ -0,0 +1,4 @@ +CREATE TEMPORARY TABLE t (i INT); +CREATE or replace TABLE t AS SELECT * FROM t; +DROP TEMPORARY TABLE t; +DROP TABLE t; diff --git a/mysql-test/suite/rpl/r/rpl_15867.result b/mysql-test/suite/rpl/r/rpl_15867.result new file mode 100644 index 00000000000..9cb63266a29 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_15867.result @@ -0,0 +1,9 @@ +include/master-slave.inc +[connection master] +CREATE TEMPORARY TABLE t (i INT); +CREATE TABLE t AS SELECT * FROM t; +connection slave; +connection master; +DROP TEMPORARY TABLE t; +DROP TABLE t; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_15867.test b/mysql-test/suite/rpl/t/rpl_15867.test new file mode 100644 index 00000000000..6de39041bb1 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_15867.test @@ -0,0 +1,11 @@ +--source include/master-slave.inc +CREATE TEMPORARY TABLE t (i INT); +CREATE TABLE t AS SELECT * FROM t; + +--sync_slave_with_master + +# Cleanup +--connection master +DROP TEMPORARY TABLE t; +DROP TABLE t; +--source include/rpl_end.inc diff --git a/mysql-test/t/create_replace_tmp.test b/mysql-test/t/create_replace_tmp.test new file mode 100644 index 00000000000..0239f4d4817 --- /dev/null +++ b/mysql-test/t/create_replace_tmp.test @@ -0,0 +1,4 @@ +CREATE TEMPORARY TABLE t (i INT); +CREATE or replace TABLE t AS SELECT * FROM t; +DROP TEMPORARY TABLE t; +DROP TABLE t; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 32e42daf7c4..0ac7a112161 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -1031,6 +1031,12 @@ retry: if (res->table && (res->table == table->table)) continue; + /* Skip if table is tmp table */ + if (check_flag & CHECK_DUP_SKIP_TEMP_TABLE && + res->table && res->table->s->tmp_table != NO_TMP_TABLE) + { + continue; + } if (check_flag & CHECK_DUP_FOR_CREATE) DBUG_RETURN(res); diff --git a/sql/sql_base.h b/sql/sql_base.h index 914cdcd4512..4f99111cbd9 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -64,6 +64,7 @@ enum find_item_error_report_type {REPORT_ALL_ERRORS, REPORT_EXCEPT_NOT_FOUND, /* Flag bits for unique_table() */ #define CHECK_DUP_ALLOW_DIFFERENT_ALIAS 1 #define CHECK_DUP_FOR_CREATE 2 +#define CHECK_DUP_SKIP_TEMP_TABLE 4 uint get_table_def_key(const TABLE_LIST *table_list, const char **key); TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type update, diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index df0ee2bd680..3ec7b54e0a9 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3910,8 +3910,8 @@ mysql_execute_command(THD *thd) { TABLE_LIST *duplicate; if ((duplicate= unique_table(thd, lex->query_tables, - lex->query_tables->next_global, - CHECK_DUP_FOR_CREATE))) + lex->query_tables->next_global, + CHECK_DUP_FOR_CREATE | CHECK_DUP_SKIP_TEMP_TABLE))) { update_non_unique_table_error(lex->query_tables, "CREATE", duplicate); From d0d073b1aaa30997307bc7aa686d3715f8c22da0 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 20 Jul 2018 19:32:28 -0700 Subject: [PATCH 18/29] Corrected and added back the test case for MDEV-15151. --- .../r/cte_recursive_not_embedded.result | 23 ++++++++++ mysql-test/t/cte_recursive_not_embedded.test | 42 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 mysql-test/r/cte_recursive_not_embedded.result create mode 100644 mysql-test/t/cte_recursive_not_embedded.test diff --git a/mysql-test/r/cte_recursive_not_embedded.result b/mysql-test/r/cte_recursive_not_embedded.result new file mode 100644 index 00000000000..202864be159 --- /dev/null +++ b/mysql-test/r/cte_recursive_not_embedded.result @@ -0,0 +1,23 @@ +# +# MDEV-15151: function with recursive CTE using no base tables +# (duplicate of MDEV-16661) +# +connection default; +CREATE TABLE t1 (id int KEY); +INSERT INTO t1 VALUES (0), (1),(2); +CREATE OR REPLACE FUNCTION func() RETURNS int +RETURN +( +WITH recursive cte AS +(SELECT 1 a UNION SELECT cte.* FROM cte natural join t1) +SELECT * FROM cte limit 1 +); +connect con1,localhost,root,,test; +SELECT func(); +connect con2,localhost,root,,test; +disconnect con2; +connection con1; +disconnect con1; +connection default; +DROP FUNCTION func; +DROP TABLE t1; diff --git a/mysql-test/t/cte_recursive_not_embedded.test b/mysql-test/t/cte_recursive_not_embedded.test new file mode 100644 index 00000000000..4dadf400681 --- /dev/null +++ b/mysql-test/t/cte_recursive_not_embedded.test @@ -0,0 +1,42 @@ +--source include/not_embedded.inc + +--echo # +--echo # MDEV-15151: function with recursive CTE using no base tables +--echo # (duplicate of MDEV-16661) +--echo # + +--connection default + +CREATE TABLE t1 (id int KEY); +INSERT INTO t1 VALUES (0), (1),(2); + +CREATE OR REPLACE FUNCTION func() RETURNS int +RETURN +( + WITH recursive cte AS + (SELECT 1 a UNION SELECT cte.* FROM cte natural join t1) + SELECT * FROM cte limit 1 +); + +--connect (con1,localhost,root,,test) + +--let $conid= `SELECT CONNECTION_ID()` +--send SELECT func() + +--connect (con2,localhost,root,,test) +--disable_query_log +--eval KILL QUERY $conid +--enable_query_log +--disconnect con2 + +--disable_result_log +--connection con1 +--error 0,ER_QUERY_INTERRUPTED +--reap +--disconnect con1 +--enable_result_log + +--connection default + +DROP FUNCTION func; +DROP TABLE t1; From b9865b289a756f68e522e094f37f24a4d7e45aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 23 Jul 2018 12:09:59 +0300 Subject: [PATCH 19/29] Follow-up to MDEV-12266: Remove latch_t::m_temp_fsp There is only one temporary tablespace. Reserving a data member in each read-write lock object for a Boolean flag seems like a huge waste, especially because this field was only actually used in debug builds. LatchDebug::check_order(): Compare to fil_system.temp_space->latch. --- storage/innobase/fil/fil0fil.cc | 1 - storage/innobase/include/sync0types.h | 24 +----------------------- storage/innobase/sync/sync0debug.cc | 18 +++++------------- 3 files changed, 6 insertions(+), 37 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 28d27890f70..7ae2a306f01 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1480,7 +1480,6 @@ fil_space_create( rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); if (space->purpose == FIL_TYPE_TEMPORARY) { - ut_d(space->latch.set_temp_fsp()); /* SysTablespace::open_or_create() would pass size!=0 to fil_node_create(), so first_time_open would not hold in fil_node_open_file(), and we diff --git a/storage/innobase/include/sync0types.h b/storage/innobase/include/sync0types.h index 73a9856e0b0..773c610d30e 100644 --- a/storage/innobase/include/sync0types.h +++ b/storage/innobase/include/sync0types.h @@ -979,8 +979,7 @@ struct latch_t { UNIV_NOTHROW : m_id(id), - m_rw_lock(), - m_temp_fsp() { } + m_rw_lock() {} /** Destructor */ virtual ~latch_t() UNIV_NOTHROW { } @@ -1014,24 +1013,6 @@ struct latch_t { return(sync_latch_get_level(m_id)); } - /** @return true if the latch is for a temporary file space*/ - bool is_temp_fsp() const - UNIV_NOTHROW - { - return(m_temp_fsp); - } - - /** Set the temporary tablespace flag. (For internal temporary - tables, MySQL 5.7 does not always acquire the index->lock. We - need to figure out the context and add some special rules - during the checks.) */ - void set_temp_fsp() - UNIV_NOTHROW - { - ut_ad(get_id() == LATCH_ID_FIL_SPACE); - m_temp_fsp = true; - } - /** @return the latch name, m_id must be set */ const char* get_name() const UNIV_NOTHROW @@ -1047,9 +1028,6 @@ struct latch_t { /** true if it is a rw-lock. In debug mode, rw_lock_t derives from this class and sets this variable. */ bool m_rw_lock; - - /** true if it is an temporary space latch */ - bool m_temp_fsp; }; /** Subclass this to iterate over a thread's acquired latch levels. */ diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 2053b36e61d..c652e6d85f2 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -35,6 +35,7 @@ Created 2012-08-21 Sunny Bains #include "ut0new.h" #include "srv0start.h" +#include "fil0fil.h" #include #include @@ -910,19 +911,10 @@ LatchDebug::check_order( case SYNC_TREE_NODE: - { - const latch_t* fsp_latch; - - fsp_latch = find(latches, SYNC_FSP); - - ut_a((fsp_latch != NULL - && fsp_latch->is_temp_fsp()) - || find(latches, SYNC_INDEX_TREE) != 0 - || find(latches, SYNC_DICT_OPERATION) - || basic_check(latches, - level, SYNC_TREE_NODE - 1)); - } - + ut_a(find(latches, SYNC_FSP) == &fil_system.temp_space->latch + || find(latches, SYNC_INDEX_TREE) + || find(latches, SYNC_DICT_OPERATION) + || basic_check(latches, level, SYNC_TREE_NODE - 1)); break; case SYNC_TREE_NODE_NEW: From 73af07536612e4f76ca24974cfa17ddd282d50a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 23 Jul 2018 13:31:10 +0300 Subject: [PATCH 20/29] Reduce the number of rw_lock_own() calls Replace pairs of rw_lock_own() calls with calls to rw_lock_own_flagged(). These calls are only present in debug builds. --- storage/innobase/btr/btr0sea.cc | 24 ++++++++++++------------ storage/innobase/buf/buf0buf.cc | 17 +++++++++-------- storage/innobase/buf/buf0lru.cc | 9 ++++----- storage/innobase/gis/gis0sea.cc | 5 ++--- storage/innobase/include/ha0ha.ic | 8 +++----- storage/innobase/include/sync0rw.ic | 3 +-- storage/innobase/row/row0log.cc | 4 ++-- storage/innobase/row/row0sel.cc | 4 ++-- storage/innobase/row/row0umod.cc | 4 ++-- storage/innobase/sync/sync0rw.cc | 8 ++++---- storage/innobase/trx/trx0i_s.cc | 22 ++++++++-------------- 11 files changed, 49 insertions(+), 59 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 7419ed2bfd5..829e0c47e39 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -529,10 +529,10 @@ btr_search_update_block_hash_info( buf_block_t* block, const btr_cur_t* cursor) { - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); - ut_ad(rw_lock_own(&block->lock, RW_LOCK_S) - || rw_lock_own(&block->lock, RW_LOCK_X)); + ut_ad(!rw_lock_own_flagged(btr_get_search_latch(cursor->index), + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); + ut_ad(rw_lock_own_flagged(&block->lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); info->last_hash_succ = FALSE; @@ -607,8 +607,8 @@ btr_search_update_hash_ref( ut_ad(cursor->flag == BTR_CUR_HASH_FAIL); ut_ad(rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); - ut_ad(rw_lock_own(&(block->lock), RW_LOCK_S) - || rw_lock_own(&(block->lock), RW_LOCK_X)); + ut_ad(rw_lock_own_flagged(&block->lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); ut_ad(page_align(btr_cur_get_rec(cursor)) == block->frame); ut_ad(page_is_leaf(block->frame)); assert_block_ahi_valid(block); @@ -667,8 +667,8 @@ btr_search_info_update_slow( buf_block_t* block; ibool build_index; - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); + ut_ad(!rw_lock_own_flagged(btr_get_search_latch(cursor->index), + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); block = btr_cur_get_block(cursor); @@ -1138,8 +1138,8 @@ retry: ut_ad(block->page.buf_fix_count == 0 || buf_block_get_state(block) == BUF_BLOCK_REMOVE_HASH - || rw_lock_own(&block->lock, RW_LOCK_S) - || rw_lock_own(&block->lock, RW_LOCK_X)); + || rw_lock_own_flagged(&block->lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); ut_ad(page_is_leaf(block->frame)); /* We must not dereference index here, because it could be freed @@ -1393,8 +1393,8 @@ btr_search_build_page_hash_index( ut_ad(page_is_leaf(block->frame)); ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); - ut_ad(rw_lock_own(&(block->lock), RW_LOCK_S) - || rw_lock_own(&(block->lock), RW_LOCK_X)); + ut_ad(rw_lock_own_flagged(&block->lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); btr_search_s_lock(index); diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 30fe1f8f616..0a5a59c9829 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4448,8 +4448,9 @@ loop: case BUF_GET_IF_IN_POOL_OR_WATCH: case BUF_PEEK_IF_IN_POOL: case BUF_EVICT_IF_IN_POOL: - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X)); - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_S)); + ut_ad(!rw_lock_own_flagged( + hash_lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); return(NULL); } @@ -4785,8 +4786,8 @@ evict_from_pool: ut_ad(block == fix_block); ut_ad(fix_block->page.buf_fix_count > 0); - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X)); - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_S)); + ut_ad(!rw_lock_own_flagged(hash_lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); ut_ad(buf_block_get_state(fix_block) == BUF_BLOCK_FILE_PAGE); @@ -4962,8 +4963,8 @@ evict_from_pool: ut_a(ibuf_count_get(fix_block->page.id) == 0); #endif - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X)); - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_S)); + ut_ad(!rw_lock_own_flagged(hash_lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); return(fix_block); } @@ -5639,8 +5640,8 @@ func_exit: ibuf_mtr_commit(&mtr); } - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X)); - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_S)); + ut_ad(!rw_lock_own_flagged(hash_lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); ut_ad(!bpage || buf_page_in_file(bpage)); return(bpage); diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 71144ee0f6e..d71d9d08511 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1630,8 +1630,8 @@ func_exit: } /* buf_LRU_block_remove_hashed() releases the hash_lock */ - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X) - && !rw_lock_own(hash_lock, RW_LOCK_S)); + ut_ad(!rw_lock_own_flagged(hash_lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); /* We have just freed a BUF_BLOCK_FILE_PAGE. If b != NULL then it was a compressed page with an uncompressed frame and @@ -2184,9 +2184,8 @@ buf_LRU_free_one_page( } /* buf_LRU_block_remove_hashed() releases hash_lock and block_mutex */ - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X) - && !rw_lock_own(hash_lock, RW_LOCK_S)); - + ut_ad(!rw_lock_own_flagged(hash_lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); ut_ad(!mutex_own(block_mutex)); } diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 173fc76ddfc..c45f2ecec59 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -258,9 +258,8 @@ rtr_pcur_getnext_from_path( rtr_info->tree_savepoints[tree_idx] = mtr_set_savepoint(mtr); #ifdef UNIV_RTR_DEBUG - ut_ad(!(rw_lock_own(&btr_cur->page_cur.block->lock, RW_LOCK_X) - || - rw_lock_own(&btr_cur->page_cur.block->lock, RW_LOCK_S)) + ut_ad(!(rw_lock_own_flagged(&btr_cur->page_cur.block->lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)) || my_latch_mode == BTR_MODIFY_TREE || my_latch_mode == BTR_CONT_MODIFY_TREE || !page_is_leaf(buf_block_get_frame( diff --git a/storage/innobase/include/ha0ha.ic b/storage/innobase/include/ha0ha.ic index 1513df209ad..3c699ab9b0c 100644 --- a/storage/innobase/include/ha0ha.ic +++ b/storage/innobase/include/ha0ha.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -137,11 +138,8 @@ hash_assert_can_search( if (table->type == HASH_TABLE_SYNC_MUTEX) { ut_ad(mutex_own(hash_get_mutex(table, fold))); } else if (table->type == HASH_TABLE_SYNC_RW_LOCK) { -# ifdef UNIV_DEBUG - rw_lock_t* lock = hash_get_lock(table, fold); - ut_ad(rw_lock_own(lock, RW_LOCK_X) - || rw_lock_own(lock, RW_LOCK_S)); -# endif + ut_ad(rw_lock_own_flagged(hash_get_lock(table, fold), + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); } else { ut_ad(table->type == HASH_TABLE_SYNC_NONE); } diff --git a/storage/innobase/include/sync0rw.ic b/storage/innobase/include/sync0rw.ic index 21872cc8bee..404c7cb9b86 100644 --- a/storage/innobase/include/sync0rw.ic +++ b/storage/innobase/include/sync0rw.ic @@ -281,8 +281,7 @@ rw_lock_s_lock_func( the threads which have s-locked a latch. This would use some CPU time. */ - ut_ad(!rw_lock_own(lock, RW_LOCK_S)); /* see NOTE above */ - ut_ad(!rw_lock_own(lock, RW_LOCK_X)); + ut_ad(!rw_lock_own_flagged(lock, RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); if (!rw_lock_s_lock_low(lock, pass, file_name, line)) { diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 55500b9533d..7e6760da372 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -290,8 +290,8 @@ row_log_online_op( ut_ad(dtuple_validate(tuple)); ut_ad(dtuple_get_n_fields(tuple) == dict_index_get_n_fields(index)); - ut_ad(rw_lock_own(dict_index_get_lock(index), RW_LOCK_S) - || rw_lock_own(dict_index_get_lock(index), RW_LOCK_X)); + ut_ad(rw_lock_own_flagged(&index->lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); if (index->is_corrupted()) { return; diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 68a81615e88..d21fd4a10ed 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -1102,8 +1102,8 @@ sel_set_rtr_rec_lock( rw_lock_x_lock(&(match->block.lock)); retry: cur_block = btr_pcur_get_block(pcur); - ut_ad(rw_lock_own(&(match->block.lock), RW_LOCK_X) - || rw_lock_own(&(match->block.lock), RW_LOCK_S)); + ut_ad(rw_lock_own_flagged(&match->block.lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); ut_ad(page_is_leaf(buf_block_get_frame(cur_block))); err = lock_sec_rec_read_check_and_lock( diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index 5f28f096286..b5c8d7c4ce2 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -265,8 +265,8 @@ row_undo_mod_clust( ut_ad(thr_get_trx(thr) == node->trx); ut_ad(node->trx->dict_operation_lock_mode); ut_ad(node->trx->in_rollback); - ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_S) - || rw_lock_own(dict_operation_lock, RW_LOCK_X)); + ut_ad(rw_lock_own_flagged(dict_operation_lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); log_free_check(); pcur = &node->pcur; diff --git a/storage/innobase/sync/sync0rw.cc b/storage/innobase/sync/sync0rw.cc index 37b64910713..c985cf9ee82 100644 --- a/storage/innobase/sync/sync0rw.cc +++ b/storage/innobase/sync/sync0rw.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1097,12 +1097,12 @@ rw_lock_own_flagged( const rw_lock_debug_t* info = *it; - ut_ad(os_thread_eq(info->thread_id, os_thread_get_curr_id())); - - if (info->pass != 0) { + if (info->pass) { continue; } + ut_ad(os_thread_eq(info->thread_id, os_thread_get_curr_id())); + switch (info->lock_type) { case RW_LOCK_S: diff --git a/storage/innobase/trx/trx0i_s.cc b/storage/innobase/trx/trx0i_s.cc index f0dc2fb78bd..363b61b0cfe 100644 --- a/storage/innobase/trx/trx0i_s.cc +++ b/storage/innobase/trx/trx0i_s.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1510,26 +1510,20 @@ cache_select_table( trx_i_s_cache_t* cache, /*!< in: whole cache */ enum i_s_table table) /*!< in: which table */ { - i_s_table_cache_t* table_cache; - - ut_ad(rw_lock_own(cache->rw_lock, RW_LOCK_S) - || rw_lock_own(cache->rw_lock, RW_LOCK_X)); + ut_ad(rw_lock_own_flagged(cache->rw_lock, + RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)); switch (table) { case I_S_INNODB_TRX: - table_cache = &cache->innodb_trx; - break; + return &cache->innodb_trx; case I_S_INNODB_LOCKS: - table_cache = &cache->innodb_locks; - break; + return &cache->innodb_locks; case I_S_INNODB_LOCK_WAITS: - table_cache = &cache->innodb_lock_waits; - break; - default: - ut_error; + return &cache->innodb_lock_waits; } - return(table_cache); + ut_error; + return NULL; } /*******************************************************************//** From b660261be1975535624eeccbe52a8b93e0aaa3bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 23 Jul 2018 15:58:31 +0300 Subject: [PATCH 21/29] ut_print_buf_hex(): Correctly dump the hex This should affect at least rec_printer() output. --- storage/innobase/ut/ut0ut.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index f655c800901..baae817f217 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -357,7 +357,7 @@ ut_print_buf_hex( for (data = static_cast(buf), i = 0; i < len; i++) { byte b = *data++; - o << hexdigit[(int) b >> 16] << hexdigit[b & 15]; + o << hexdigit[int(b) >> 4] << hexdigit[b & 15]; } o << ")"; From 730f6c912c5746e405c62f6c28b571771de0f5bf Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 23 Jul 2018 13:42:19 +0300 Subject: [PATCH 22/29] MDEV-16779 Assertion !rw_lock_own failed upon purge This is a regression caused by the fix of MDEV-15855. purge_vcol_info_t::set_used(): Add a missing condition. row_purge_poss_sec(): Invoke set_used() in order to have !is_first_fetch() when retrying. --- storage/innobase/include/row0types.h | 4 +++- storage/innobase/row/row0purge.cc | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/innobase/include/row0types.h b/storage/innobase/include/row0types.h index 1bff3767253..005dc2ad95f 100644 --- a/storage/innobase/include/row0types.h +++ b/storage/innobase/include/row0types.h @@ -92,7 +92,9 @@ struct purge_vcol_info_t return; } - first_use = used = true; + if (!used) { + first_use = used = true; + } } /** Check whether it fetches mariadb table for the first time. diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 8fe0a59eb3c..59387f926ff 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -345,6 +345,7 @@ retry_purge_sec: if (node->vcol_info.is_first_fetch()) { if (node->vcol_info.mariadb_table) { + node->vcol_info.set_used(); goto retry_purge_sec; } From a7a0c533c20531aa658941a885a61b96f22110b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 23 Jul 2018 17:50:56 +0300 Subject: [PATCH 23/29] Follow-up to MDEV-15855: Remove bogus debug assertions During a table-rebuilding operation, the function table_name_parse() can encounter a table name that starts with #sql. Here is an example of a failure: CURRENT_TEST: gcol.innodb_virtual_basic mysqltest: At line 1204: query 'alter table t drop column d ' failed: 2013: Lost connection to MySQL server during query Let us just remove these bogus debug assertions. If the final renaming phase during ALTER TABLE never fails, it should not do any harm to skip the purge. If it does fail, then we might end up 'leaking' some delete-marked records in the indexes on virtual columns of the original table, and these garbage records would keep consuming space until the indexes are dropped or the table is successfully rebuilt. --- storage/innobase/handler/ha_innodb.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index deb6f05e853..7000865a7cd 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -21661,7 +21661,6 @@ static TABLE* innodb_acquire_mdl(THD* thd, dict_table_t* table) if (!table_name_parse(table->name, db_buf, tbl_buf, db_buf_len, tbl_buf_len)) { - ut_ad(!"invalid table name"); return NULL; } @@ -21697,7 +21696,6 @@ fail: if (!table_name_parse(table->name, db_buf1, tbl_buf1, db_buf1_len, tbl_buf1_len)) { - ut_ad(!"invalid table name"); goto release_fail; } @@ -21745,7 +21743,6 @@ static TABLE* innodb_find_table_for_vc(THD* thd, dict_table_t* table) if (!table_name_parse(table->name, db_buf, tbl_buf, db_buf_len, tbl_buf_len)) { - ut_ad(!"invalid table name"); return NULL; } From c5ba13dda0464b1316bfb45efbcdcc924817f0de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 23 Jul 2018 18:23:54 +0300 Subject: [PATCH 24/29] MDEV-15855 cleanup: Privatize purge_vcol_info_t Declare all fields of purge_vcol_info_t private, and add accessor functions. --- storage/innobase/include/row0types.h | 36 ++++++++++++++++++++++++++++ storage/innobase/row/row0purge.cc | 16 +++++++------ storage/innobase/row/row0vers.cc | 11 ++++----- 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/storage/innobase/include/row0types.h b/storage/innobase/include/row0types.h index 005dc2ad95f..d2aef89f695 100644 --- a/storage/innobase/include/row0types.h +++ b/storage/innobase/include/row0types.h @@ -56,6 +56,7 @@ struct TABLE; /** Purge virtual column node information. */ struct purge_vcol_info_t { +private: /** Is there a possible need to evaluate virtual columns? */ bool requested; /** Do we have to evaluate virtual columns (using mariadb_table)? */ @@ -67,6 +68,7 @@ struct purge_vcol_info_t /** MariaDB table opened for virtual column computation. */ TABLE* mariadb_table; +public: /** Reset the state. */ void reset() { @@ -81,6 +83,29 @@ struct purge_vcol_info_t or doesn't try to calculate virtual column. */ bool validate() const { return !used || mariadb_table; } + /** @return the table handle for evaluating virtual columns */ + TABLE* table() const { return mariadb_table; } + + /** Set the table handle for evaluating virtual columns. + @param[in] table table handle */ + void set_table(TABLE* table) + { + ut_ad(!table || is_first_fetch()); + mariadb_table = table; + } + + /** Note that virtual column information may be needed. */ + void set_requested() + { + ut_ad(!used); + ut_ad(!first_use); + ut_ad(!mariadb_table); + requested = true; + } + + /** @return whether the virtual column information may be needed */ + bool is_requested() const { return requested; } + /** Note that the virtual column information is needed. */ void set_used() { @@ -97,11 +122,22 @@ struct purge_vcol_info_t } } + /** @return whether the virtual column information is needed */ + bool is_used() const + { + ut_ad(!first_use || used); + ut_ad(!used || requested); + ut_ad(used || !mariadb_table); + return used; + } + /** Check whether it fetches mariadb table for the first time. @return true if first time tries to open mariadb table. */ bool is_first_fetch() const { ut_ad(!first_use || used); + ut_ad(!used || requested); + ut_ad(used || !mariadb_table); return first_use; } }; diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 59387f926ff..cc2dcc9b798 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -137,7 +137,7 @@ row_purge_remove_clust_if_poss_low( rec_offs_init(offsets_); ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_S) - || node->vcol_info.used); + || node->vcol_info.is_used()); index = dict_table_get_first_index(node->table); @@ -250,7 +250,7 @@ static void row_purge_store_vsec_cur( return; } - node->vcol_info.requested = true; + node->vcol_info.set_requested(); btr_pcur_store_position(sec_pcur, sec_mtr); @@ -318,7 +318,7 @@ row_purge_poss_sec( ut_ad(!dict_index_is_clust(index)); - const bool store_cur = sec_mtr && !node->vcol_info.used + const bool store_cur = sec_mtr && !node->vcol_info.is_used() && dict_index_has_virtual(index); if (store_cur) { @@ -327,7 +327,7 @@ row_purge_poss_sec( /* The PRIMARY KEY value was not found in the clustered index. The secondary index record found. We can purge the secondary index record. */ - if (!node->vcol_info.requested) { + if (!node->vcol_info.is_requested()) { ut_ad(!node->found_clust); return true; } @@ -344,7 +344,9 @@ retry_purge_sec: &node->vcol_info); if (node->vcol_info.is_first_fetch()) { - if (node->vcol_info.mariadb_table) { + ut_ad(store_cur); + + if (node->vcol_info.table()) { node->vcol_info.set_used(); goto retry_purge_sec; } @@ -801,7 +803,7 @@ row_purge_upd_exist_or_extern_func( mem_heap_t* heap; ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_S) - || node->vcol_info.used); + || node->vcol_info.is_used()); ut_ad(!node->table->skip_alter_undo); if (node->rec_type == TRX_UNDO_UPD_DEL_REC @@ -1139,7 +1141,7 @@ row_purge( bool purged = row_purge_record( node, undo_rec, thr, updated_extern); - if (!node->vcol_info.used) { + if (!node->vcol_info.is_used()) { rw_lock_s_unlock(dict_operation_lock); } diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 9ced22fd54b..f58f0a47bd5 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -457,7 +457,7 @@ row_vers_build_clust_v_col( if (vcol_info != NULL) { vcol_info->set_used(); - maria_table = vcol_info->mariadb_table; + maria_table = vcol_info->table(); } innobase_allocate_row_for_vcol(thd, index, @@ -466,9 +466,8 @@ row_vers_build_clust_v_col( &record, &vcol_storage); - if (vcol_info && !vcol_info->mariadb_table) { - vcol_info->mariadb_table = maria_table; - ut_ad(!maria_table || vcol_info->is_first_fetch()); + if (vcol_info && !vcol_info->table()) { + vcol_info->set_table(maria_table); goto func_exit; } @@ -834,7 +833,7 @@ row_vers_build_cur_vrow( rec, *clust_offsets, NULL, NULL, NULL, NULL, heap); - if (vcol_info && !vcol_info->used) { + if (vcol_info && !vcol_info->is_used()) { mtr->commit(); } @@ -955,7 +954,7 @@ row_vers_old_has_index_entry( if (trx_undo_roll_ptr_is_insert(t_roll_ptr) || dbug_v_purge) { - if (vcol_info && !vcol_info->used) { + if (vcol_info && !vcol_info->is_used()) { mtr->commit(); } From 9d1f3bf2e9425db8e352ee80e7b456dd6ef73fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 23 Jul 2018 18:23:54 +0300 Subject: [PATCH 25/29] row_purge_poss_sec(): Add debug instrumentation This helped fix MDEV-16779. --- storage/innobase/row/row0purge.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index cc2dcc9b798..7e7dd794f1b 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -323,6 +323,8 @@ row_purge_poss_sec( if (store_cur) { row_purge_store_vsec_cur(node, index, sec_pcur, sec_mtr); + ut_ad(sec_mtr->has_committed() + == node->vcol_info.is_requested()); /* The PRIMARY KEY value was not found in the clustered index. The secondary index record found. We can purge @@ -346,7 +348,15 @@ retry_purge_sec: if (node->vcol_info.is_first_fetch()) { ut_ad(store_cur); - if (node->vcol_info.table()) { + const TABLE* t= node->vcol_info.table(); + DBUG_LOG("purge", "retry " << t + << (is_tree ? " tree" : " leaf") + << index->name << "," << index->table->name + << ": " << rec_printer(entry).str()); + + ut_ad(mtr.has_committed()); + + if (t) { node->vcol_info.set_used(); goto retry_purge_sec; } @@ -360,9 +370,11 @@ retry_purge_sec: if (node->found_clust) { btr_pcur_commit_specify_mtr(&node->pcur, &mtr); } else { - mtr_commit(&mtr); + mtr.commit(); } + ut_ad(mtr.has_committed()); + if (store_cur && !row_purge_restore_vsec_cur( node, index, sec_pcur, sec_mtr, is_tree)) { return false; From a86a02a8447233940cc3b0f40e2b403691997807 Mon Sep 17 00:00:00 2001 From: Jacob Mathew Date: Fri, 20 Jul 2018 18:21:32 -0700 Subject: [PATCH 26/29] MDEV-15786: ERROR 1062 (23000) at line 365: Duplicate entry 'spider' for key 'PRIMARY' The problem occurs on Ubuntu where a Spider package is installed on the system separately from the MariaDB package. MariaDB and Spider upgrades leave the Spider plugin improperly installed. Spider is present in the mysql.plugin table but is not present in information_schema. The problem has been corrected in Spider's installation script. Logic has been added to check for Spider entries in both information_schema and mysql.plugin. If Spider is present in mysql.plugin but is not present in information_schema, then Spider is first removed from mysql.plugin. The subsequent plugin install of Spider will insert entries in both mysql.plugin and information_schema. Author: Jacob Mathew. Reviewer: Kentoku Shiba. Cherry-Picked: Commit 0897d81 on branch bb-10.3-MDEV-15786 --- storage/spider/scripts/install_spider.sql | 24 ++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/storage/spider/scripts/install_spider.sql b/storage/spider/scripts/install_spider.sql index c6581d331f3..c5a86caa219 100644 --- a/storage/spider/scripts/install_spider.sql +++ b/storage/spider/scripts/install_spider.sql @@ -412,9 +412,18 @@ delimiter // create procedure mysql.spider_plugin_installer() begin set @win_plugin := IF(@@version_compile_os like 'Win%', 1, 0); + set @have_spider_i_s_plugin := 0; + select @have_spider_i_s_plugin := 1 from INFORMATION_SCHEMA.plugins where PLUGIN_NAME = 'SPIDER'; set @have_spider_plugin := 0; - select @have_spider_plugin := 1 from INFORMATION_SCHEMA.plugins where PLUGIN_NAME = 'SPIDER'; - if @have_spider_plugin = 0 then + select @have_spider_plugin := 1 from mysql.plugin where name = 'spider'; + if @have_spider_i_s_plugin = 0 then + if @have_spider_plugin = 1 then + -- spider plugin is present in mysql.plugin but not in + -- information_schema.plugins. Remove spider plugin entry + -- in mysql.plugin first. + delete from mysql.plugin where name = 'spider'; + end if; + -- Install spider plugin if @win_plugin = 0 then install plugin spider soname 'ha_spider.so'; else @@ -423,7 +432,16 @@ begin end if; set @have_spider_i_s_alloc_mem_plugin := 0; select @have_spider_i_s_alloc_mem_plugin := 1 from INFORMATION_SCHEMA.plugins where PLUGIN_NAME = 'SPIDER_ALLOC_MEM'; - if @have_spider_i_s_alloc_mem_plugin = 0 then + set @have_spider_alloc_mem_plugin := 0; + select @have_spider_alloc_mem_plugin := 1 from mysql.plugin where name = 'spider_alloc_mem'; + if @have_spider_i_s_alloc_mem_plugin = 0 then + if @have_spider_alloc_mem_plugin = 1 then + -- spider_alloc_mem plugin is present in mysql.plugin but not in + -- information_schema.plugins. Remove spider_alloc_mem plugin entry + -- in mysql.plugin first. + delete from mysql.plugin where name = 'spider_alloc_mem'; + end if; + -- Install spider_alloc_mem plugin if @win_plugin = 0 then install plugin spider_alloc_mem soname 'ha_spider.so'; else From 45ab00f097be0f77d7087182244218f036c3f113 Mon Sep 17 00:00:00 2001 From: Jacob Mathew Date: Mon, 23 Jul 2018 14:14:23 -0700 Subject: [PATCH 27/29] MDEV-15786: ERROR 1062 (23000) at line 365: Duplicate entry 'spider' for key 'PRIMARY' The problem occurs on Ubuntu where a Spider package is installed on the system separately from the MariaDB package. MariaDB and Spider upgrades leave the Spider plugin improperly installed. Spider is present in the mysql.plugin table but is not present in information_schema. The problem has been corrected in Spider's installation script. Logic has been added to check for Spider entries in both information_schema and mysql.plugin. If Spider is present in mysql.plugin but is not present in information_schema, then Spider is first removed from mysql.plugin. The subsequent plugin install of Spider will insert entries in both mysql.plugin and information_schema. Author: Jacob Mathew. Reviewer: Kentoku Shiba. Cherry-Picked: Commit 0897d81 on branch bb-10.3-MDEV-15786 --- storage/spider/scripts/install_spider.sql | 24 ++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/storage/spider/scripts/install_spider.sql b/storage/spider/scripts/install_spider.sql index dbafe95bdbf..9c9eaacdf60 100644 --- a/storage/spider/scripts/install_spider.sql +++ b/storage/spider/scripts/install_spider.sql @@ -297,9 +297,18 @@ delimiter // create procedure mysql.spider_plugin_installer() begin set @win_plugin := IF(@@version_compile_os like 'Win%', 1, 0); + set @have_spider_i_s_plugin := 0; + select @have_spider_i_s_plugin := 1 from INFORMATION_SCHEMA.plugins where PLUGIN_NAME = 'SPIDER'; set @have_spider_plugin := 0; - select @have_spider_plugin := 1 from INFORMATION_SCHEMA.plugins where PLUGIN_NAME = 'SPIDER'; - if @have_spider_plugin = 0 then + select @have_spider_plugin := 1 from mysql.plugin where name = 'spider'; + if @have_spider_i_s_plugin = 0 then + if @have_spider_plugin = 1 then + -- spider plugin is present in mysql.plugin but not in + -- information_schema.plugins. Remove spider plugin entry + -- in mysql.plugin first. + delete from mysql.plugin where name = 'spider'; + end if; + -- Install spider plugin if @win_plugin = 0 then install plugin spider soname 'ha_spider.so'; else @@ -308,7 +317,16 @@ begin end if; set @have_spider_i_s_alloc_mem_plugin := 0; select @have_spider_i_s_alloc_mem_plugin := 1 from INFORMATION_SCHEMA.plugins where PLUGIN_NAME = 'SPIDER_ALLOC_MEM'; - if @have_spider_i_s_alloc_mem_plugin = 0 then + set @have_spider_alloc_mem_plugin := 0; + select @have_spider_alloc_mem_plugin := 1 from mysql.plugin where name = 'spider_alloc_mem'; + if @have_spider_i_s_alloc_mem_plugin = 0 then + if @have_spider_alloc_mem_plugin = 1 then + -- spider_alloc_mem plugin is present in mysql.plugin but not in + -- information_schema.plugins. Remove spider_alloc_mem plugin entry + -- in mysql.plugin first. + delete from mysql.plugin where name = 'spider_alloc_mem'; + end if; + -- Install spider_alloc_mem plugin if @win_plugin = 0 then install plugin spider_alloc_mem soname 'ha_spider.so'; else From f74d2a9faa4b4233433dfcb0d2d14a6b269c48ba Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 24 Jul 2018 13:19:31 +0400 Subject: [PATCH 28/29] MDEV-16729 VARCHAR COMPRESSED is created with a wrong length for multi-byte character sets Field_varstring::sql_type() did not calculate character length correctly. Using char_length() instead of the bad code. --- mysql-test/main/column_compression.result | 14 ++++++++++++++ mysql-test/main/column_compression.test | 11 +++++++++++ sql/field.cc | 5 ++--- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/mysql-test/main/column_compression.result b/mysql-test/main/column_compression.result index ace65387181..c783e7fd1e9 100644 --- a/mysql-test/main/column_compression.result +++ b/mysql-test/main/column_compression.result @@ -1472,3 +1472,17 @@ SELECT LEFT(a, 10), LENGTH(a) FROM t1; LEFT(a, 10) LENGTH(a) aaaaaaaaaa 255 DROP TABLE t1; +# +# MDEV-16729 VARCHAR COMPRESSED is created with a wrong length for multi-byte character sets +# +CREATE OR REPLACE TABLE t1 (a VARCHAR(1000) CHARACTER SET utf8 COMPRESSED); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(1000) /*!100301 COMPRESSED*/ CHARACTER SET utf8 DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +SELECT COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +COLUMN_TYPE +varchar(1000) /*!100301 COMPRESSED*/ +DROP TABLE t1; diff --git a/mysql-test/main/column_compression.test b/mysql-test/main/column_compression.test index f3220503694..c9d0014ab9e 100644 --- a/mysql-test/main/column_compression.test +++ b/mysql-test/main/column_compression.test @@ -170,3 +170,14 @@ CREATE TABLE t1(a TINYTEXT COMPRESSED); INSERT INTO t1 VALUES(REPEAT('a', 255)); SELECT LEFT(a, 10), LENGTH(a) FROM t1; DROP TABLE t1; + + +--echo # +--echo # MDEV-16729 VARCHAR COMPRESSED is created with a wrong length for multi-byte character sets +--echo # + +CREATE OR REPLACE TABLE t1 (a VARCHAR(1000) CHARACTER SET utf8 COMPRESSED); +SHOW CREATE TABLE t1; +SELECT COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; diff --git a/sql/field.cc b/sql/field.cc index efc20fa37fc..4c0f1b0560b 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7715,10 +7715,9 @@ void Field_varstring::sql_type(String &res) const size_t length; length= cs->cset->snprintf(cs,(char*) res.ptr(), - res.alloced_length(), "%s(%d)", + res.alloced_length(), "%s(%u)", (has_charset() ? "varchar" : "varbinary"), - (int) field_length / charset()->mbmaxlen - - MY_TEST(compression_method())); + (uint) char_length()); res.length(length); if ((thd->variables.sql_mode & (MODE_MYSQL323 | MODE_MYSQL40)) && has_charset() && (charset()->state & MY_CS_BINSORT)) From a8227a154306a818685ab291f1715c3971d03099 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 24 Jul 2018 18:15:15 +0400 Subject: [PATCH 29/29] MDEV-16814 CREATE TABLE SELECT JSON_QUOTE(multibyte_charset_expr) makes a field of a wrong length --- mysql-test/r/func_json.result | 15 ++++++++++++++- mysql-test/t/func_json.test | 9 +++++++++ sql/item_jsonfunc.cc | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result index 564442a2880..39bd46b7dea 100644 --- a/mysql-test/r/func_json.result +++ b/mysql-test/r/func_json.result @@ -297,7 +297,7 @@ json_quote('foo') show create table t1; Table Create Table t1 CREATE TABLE `t1` ( - `json_quote('foo')` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + `json_quote('foo')` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 drop table t1; select json_merge('string'); @@ -747,3 +747,16 @@ INSERT INTO t1 VALUES ('foo'),('bar'); SELECT * FROM t1 WHERE c IN (JSON_EXTRACT('{"a":"b"}', '$.*')); c DROP TABLE t1; +# +# MDEV-16814 CREATE TABLE SELECT JSON_QUOTE(multibyte_charset_expr) makes a field of a wrong length +# +CREATE TABLE t1 AS SELECT +JSON_QUOTE(_latin1'foo') AS c1, +JSON_QUOTE(_utf8'foo') AS c2; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `c2` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test index 1b91061b9a3..37b18763e91 100644 --- a/mysql-test/t/func_json.test +++ b/mysql-test/t/func_json.test @@ -413,3 +413,12 @@ INSERT INTO t1 VALUES ('foo'),('bar'); SELECT * FROM t1 WHERE c IN (JSON_EXTRACT('{"a":"b"}', '$.*')); DROP TABLE t1; +--echo # +--echo # MDEV-16814 CREATE TABLE SELECT JSON_QUOTE(multibyte_charset_expr) makes a field of a wrong length +--echo # + +CREATE TABLE t1 AS SELECT + JSON_QUOTE(_latin1'foo') AS c1, + JSON_QUOTE(_utf8'foo') AS c2; +SHOW CREATE TABLE t1; +DROP TABLE t1; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 8103aa906ff..ac0d7e9e6b0 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -575,7 +575,7 @@ bool Item_func_json_quote::fix_length_and_dec() Odd but realistic worst case is when all characters of the argument turn into '\uXXXX\uXXXX', which is 12. */ - max_length= args[0]->max_length * 12 + 2; + fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * 12 + 2); return FALSE; }