From a9a4089175a8fd5a9b973f99f87cd0fe181f8f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 14 Dec 2017 13:47:38 +0200 Subject: [PATCH 01/23] Plug a small memory leak in mariabackup --backup --- extra/mariabackup/xtrabackup.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 90a0c829773..f145a94c923 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -3626,7 +3626,6 @@ fail: /* Reset the system variables in the recovery module. */ recv_sys_var_init(); trx_pool_init(); - row_mysql_init(); ut_crc32_init(); crc_init(); From ef9e78c9d41a5ac644f08068e3dabad948b0e30a Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 13 Dec 2017 11:52:53 +0100 Subject: [PATCH 02/23] MDEV-14524 TokuDB is unable to be built on Linux tokudb needs either F_NOCACHE or O_DIRECT, not both --- storage/tokudb/CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 4e0291545ec..f01c4aed449 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -4,13 +4,17 @@ IF(CMAKE_VERSION VERSION_LESS "2.8.9") MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB") ELSEIF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") -# tokudb requires F_NOCACHE, O_DIRECT, and designated initializers +# tokudb requires F_NOCACHE or O_DIRECT, and designated initializers CHECK_CXX_SOURCE_COMPILES( " #include struct a {int b; int c; }; struct a d = { .b=1, .c=2 }; -int main() { return F_NOCACHE + O_DIRECT; } +#if defined(O_DIRECT) || defined(F_NOCACHE) +int main() { return 0; } +#else +#error +#endif " TOKUDB_OK) ENDIF() From 64b11e61b5055cbdcdb97911d426b4f13f14fbd1 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 15 Dec 2017 17:59:33 +0300 Subject: [PATCH 03/23] MDEV-14293: MyRocks lacks basic functionality The error "Unsupported collation on string indexed column %s Use binary collation (latin1_bin, binary, utf8_bin)." is misleading. Change it: - It is now a warning - It is printed only for collations that do not support index-only access (reversible collations that use unpack_info are ok) - The new warning text is: Indexed column %s.%s uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key. --- sql/share/errmsg-utf8.txt | 3 +++ storage/rocksdb/ha_rocksdb.cc | 22 +++++++------------ .../rocksdb/r/add_index_inplace.result | 9 +++++++- .../rocksdb/t/add_index_inplace.test | 9 +++++++- storage/rocksdb/rdb_datadic.cc | 2 +- storage/rocksdb/rdb_datadic.h | 2 ++ 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 528dc7107e6..50ccc6eadd1 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -7753,3 +7753,6 @@ ER_GEOJSON_EMPTY_COORDINATES ER_MYROCKS_CANT_NOPAD_COLLATION eng "MyRocks doesn't currently support collations with \"No pad\" attribute." +ER_MYROCKS_COLLATION_IS_LIMITED + eng "Indexed column %s.%s uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key." + diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index a9d37c1def2..6afba00b3f8 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -5957,9 +5957,11 @@ rdb_is_index_collation_supported(const my_core::Field *const field) { const my_core::enum_field_types type = field->real_type(); /* Handle [VAR](CHAR|BINARY) or TEXT|BLOB */ if (type == MYSQL_TYPE_VARCHAR || type == MYSQL_TYPE_STRING || - type == MYSQL_TYPE_BLOB) { - return RDB_INDEX_COLLATIONS.find(field->charset()->number) != - RDB_INDEX_COLLATIONS.end(); + type == MYSQL_TYPE_BLOB) { + + return (RDB_INDEX_COLLATIONS.find(field->charset()->number) != + RDB_INDEX_COLLATIONS.end()) || + rdb_is_collation_supported(field->charset()); } return true; } @@ -6118,18 +6120,10 @@ int ha_rocksdb::create_cfs( !rdb_is_index_collation_supported( table_arg->key_info[i].key_part[part].field) && !rdb_collation_exceptions->matches(tablename_sys)) { - std::string collation_err; - for (const auto &coll : RDB_INDEX_COLLATIONS) { - if (collation_err != "") { - collation_err += ", "; - } - collation_err += get_charset_name(coll); - } - my_error(ER_UNSUPPORTED_COLLATION, MYF(0), + + my_error(ER_MYROCKS_COLLATION_IS_LIMITED, MYF(ME_JUST_WARNING), tbl_def_arg->full_tablename().c_str(), - table_arg->key_info[i].key_part[part].field->field_name, - collation_err.c_str()); - DBUG_RETURN(HA_EXIT_FAILURE); + table_arg->key_info[i].key_part[part].field->field_name); } } } diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result index 01fa9f1d35b..ca770c2358a 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result @@ -279,8 +279,15 @@ DROP TABLE t1; set @tmp_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check; set global rocksdb_strict_collation_check=1; CREATE TABLE t1 (a INT, b TEXT); +# MariaDB no longer gives ER_UNSUPPORTED_COLLATION ALTER TABLE t1 ADD KEY kb(b(10)); -ERROR HY000: Unsupported collation on string indexed column test.t1.b Use binary collation (latin1_bin, binary, utf8_bin). +ALTER TABLE t1 ADD PRIMARY KEY(a); +DROP TABLE t1; +CREATE TABLE t1 (a INT, b TEXT collate utf8_general_ci); +# MariaDB no longer gives ER_UNSUPPORTED_COLLATION +ALTER TABLE t1 ADD KEY kb(b(10)); +Warnings: +Warning 4078 Indexed column test.t1.b uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key. ALTER TABLE t1 ADD PRIMARY KEY(a); DROP TABLE t1; set global rocksdb_strict_collation_check= @tmp_rocksdb_strict_collation_check; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test index c1a91c2a5a2..876ef2c9965 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test @@ -173,10 +173,17 @@ set @tmp_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check; set global rocksdb_strict_collation_check=1; CREATE TABLE t1 (a INT, b TEXT); ---error ER_UNSUPPORTED_COLLATION +--echo # MariaDB no longer gives ER_UNSUPPORTED_COLLATION ALTER TABLE t1 ADD KEY kb(b(10)); ALTER TABLE t1 ADD PRIMARY KEY(a); DROP TABLE t1; + +CREATE TABLE t1 (a INT, b TEXT collate utf8_general_ci); +--echo # MariaDB no longer gives ER_UNSUPPORTED_COLLATION +ALTER TABLE t1 ADD KEY kb(b(10)); +ALTER TABLE t1 ADD PRIMARY KEY(a); +DROP TABLE t1; + set global rocksdb_strict_collation_check= @tmp_rocksdb_strict_collation_check; # make sure race condition between connection close and alter on another diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index 3efd3ae6433..df743ff0c48 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -2921,7 +2921,7 @@ std::array rdb_collation_data; mysql_mutex_t rdb_collation_data_mutex; -static bool rdb_is_collation_supported(const my_core::CHARSET_INFO *const cs) { +bool rdb_is_collation_supported(const my_core::CHARSET_INFO *const cs) { return cs->strxfrm_multiply==1 && cs->mbmaxlen == 1 && !(cs->state & (MY_CS_BINSORT | MY_CS_NOPAD)); } diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index b1ecef045d1..5796132de39 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -1358,4 +1358,6 @@ struct Rdb_index_info { uint64 m_ttl_duration = 0; }; +bool rdb_is_collation_supported(const my_core::CHARSET_INFO *const cs); + } // namespace myrocks From 7380376370dd9ec93fbcd684c80ab4e5bda70f24 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 16 Dec 2017 16:44:33 +0300 Subject: [PATCH 04/23] MDEV-14293: MyRocks lacks basic functionality Adjust the fix according to the review input: don't introduce a special error caode, use ER_INTERNAL_ERROR catch-all. --- sql/share/errmsg-utf8.txt | 4 ---- storage/rocksdb/ha_rocksdb.cc | 12 +++++++++--- .../mysql-test/rocksdb/r/add_index_inplace.result | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 50ccc6eadd1..74ac7e9f557 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -7752,7 +7752,3 @@ ER_GEOJSON_EMPTY_COORDINATES ER_MYROCKS_CANT_NOPAD_COLLATION eng "MyRocks doesn't currently support collations with \"No pad\" attribute." - -ER_MYROCKS_COLLATION_IS_LIMITED - eng "Indexed column %s.%s uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key." - diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 6afba00b3f8..766c8036fba 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -6121,9 +6121,15 @@ int ha_rocksdb::create_cfs( table_arg->key_info[i].key_part[part].field) && !rdb_collation_exceptions->matches(tablename_sys)) { - my_error(ER_MYROCKS_COLLATION_IS_LIMITED, MYF(ME_JUST_WARNING), - tbl_def_arg->full_tablename().c_str(), - table_arg->key_info[i].key_part[part].field->field_name); + char buf[1024]; + my_snprintf(buf, sizeof(buf), + "Indexed column %s.%s uses a collation that does not " + "allow index-only access in secondary key and has " + "reduced disk space efficiency in primary key.", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name); + + my_error(ER_INTERNAL_ERROR, MYF(ME_JUST_WARNING), buf); } } } diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result index ca770c2358a..6325dc97cf5 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result @@ -287,7 +287,7 @@ CREATE TABLE t1 (a INT, b TEXT collate utf8_general_ci); # MariaDB no longer gives ER_UNSUPPORTED_COLLATION ALTER TABLE t1 ADD KEY kb(b(10)); Warnings: -Warning 4078 Indexed column test.t1.b uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key. +Warning 1815 Internal error: Indexed column test.t1.b uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key. ALTER TABLE t1 ADD PRIMARY KEY(a); DROP TABLE t1; set global rocksdb_strict_collation_check= @tmp_rocksdb_strict_collation_check; From 4bd63bd55158566db8c8b46b29fee590e9596a5e Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 16 Dec 2017 17:45:55 +0300 Subject: [PATCH 05/23] MDEV-14679: RocksdB plugin fails to load with "Loading of unknown plugin ROCKSDB_CFSTATS Set maturity level of all "Sub-plugins" to be the same as the main maturity level. --- storage/rocksdb/ha_rocksdb.cc | 3 ++- storage/rocksdb/ha_rocksdb.h | 3 +++ .../rocksdb/r/mariadb_port_fixes.result | 17 ++++++++++++++ .../rocksdb/t/mariadb_port_fixes.test | 6 +++++ storage/rocksdb/rdb_i_s.cc | 22 +++++++++---------- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 766c8036fba..98a9785713a 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -12499,6 +12499,7 @@ void print_keydup_error(TABLE *table, KEY *key, myf errflag, its name generation. */ + struct st_mysql_storage_engine rocksdb_storage_engine = { MYSQL_HANDLERTON_INTERFACE_VERSION}; @@ -12515,7 +12516,7 @@ maria_declare_plugin(rocksdb_se){ myrocks::rocksdb_status_vars, /* status variables */ myrocks::rocksdb_system_variables, /* system variables */ "1.0", /* string version */ - MariaDB_PLUGIN_MATURITY_ALPHA /* maturity */ + myrocks::MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }, myrocks::rdb_i_s_cfstats, myrocks::rdb_i_s_dbstats, myrocks::rdb_i_s_perf_context, myrocks::rdb_i_s_perf_context_global, diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index d064988d0f3..82819bbf7b2 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -1411,4 +1411,7 @@ private: Rdb_inplace_alter_ctx(const Rdb_inplace_alter_ctx &); Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &); }; + +const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_ALPHA; + } // namespace myrocks diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index f590fd22dff..9952314cd2c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -64,3 +64,20 @@ set global rocksdb_strict_collation_check=off; create table t1 (pk varchar(10) collate latin1_nopad_bin, primary key(pk)) engine=rocksdb; ERROR HY000: MyRocks doesn't currently support collations with "No pad" attribute. set global rocksdb_strict_collation_check=@tmp_rscc; +# +# MDEV-14679: RocksdB plugin fails to load with "Loading of unknown plugin ROCKSDB_CFSTATS +# +select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; +plugin_name plugin_maturity +ROCKSDB Alpha +ROCKSDB_CFSTATS Alpha +ROCKSDB_DBSTATS Alpha +ROCKSDB_PERF_CONTEXT Alpha +ROCKSDB_PERF_CONTEXT_GLOBAL Alpha +ROCKSDB_CF_OPTIONS Alpha +ROCKSDB_COMPACTION_STATS Alpha +ROCKSDB_GLOBAL_INFO Alpha +ROCKSDB_DDL Alpha +ROCKSDB_INDEX_FILE_MAP Alpha +ROCKSDB_LOCKS Alpha +ROCKSDB_TRX Alpha diff --git a/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test b/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test index 70a4f5b05cb..980f2e302b2 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test @@ -67,3 +67,9 @@ set global rocksdb_strict_collation_check=off; create table t1 (pk varchar(10) collate latin1_nopad_bin, primary key(pk)) engine=rocksdb; set global rocksdb_strict_collation_check=@tmp_rscc; + +--echo # +--echo # MDEV-14679: RocksdB plugin fails to load with "Loading of unknown plugin ROCKSDB_CFSTATS +--echo # +select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; + diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index 346a2994ec1..424a9e6c1f4 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -1484,7 +1484,7 @@ struct st_maria_plugin rdb_i_s_cfstats = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_dbstats = { @@ -1500,7 +1500,7 @@ struct st_maria_plugin rdb_i_s_dbstats = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_perf_context = { @@ -1516,7 +1516,7 @@ struct st_maria_plugin rdb_i_s_perf_context = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_perf_context_global = { @@ -1532,7 +1532,7 @@ struct st_maria_plugin rdb_i_s_perf_context_global = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_cfoptions = { @@ -1548,7 +1548,7 @@ struct st_maria_plugin rdb_i_s_cfoptions = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_global_info = { @@ -1564,7 +1564,7 @@ struct st_maria_plugin rdb_i_s_global_info = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_compact_stats = { @@ -1580,7 +1580,7 @@ struct st_maria_plugin rdb_i_s_compact_stats = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_ddl = { @@ -1596,7 +1596,7 @@ struct st_maria_plugin rdb_i_s_ddl = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_index_file_map = { @@ -1612,7 +1612,7 @@ struct st_maria_plugin rdb_i_s_index_file_map = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_lock_info = { @@ -1628,7 +1628,7 @@ struct st_maria_plugin rdb_i_s_lock_info = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_trx_info = { @@ -1644,6 +1644,6 @@ struct st_maria_plugin rdb_i_s_trx_info = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; } // namespace myrocks From 06f0b23a78ab15b6f3d4465e77ac1bdc747524d4 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 17 Dec 2017 17:53:53 +0200 Subject: [PATCH 06/23] Fixed memory leak in my_rocks rocksdb_sys_vars.rocksdb_update_cf_options_basic.test failed with reports about not freed memory. --- storage/rocksdb/ha_rocksdb.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 98a9785713a..8857b2e8cba 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -12337,6 +12337,7 @@ void rocksdb_set_update_cf_options(THD *const /* unused */, // Basic sanity checking and parsing the options into a map. If this fails // then there's no point to proceed. if (!Rdb_cf_options::parse_cf_options(val, &option_map)) { + my_free(*reinterpret_cast(var_ptr)); *reinterpret_cast(var_ptr) = nullptr; // NO_LINT_DEBUG @@ -12405,6 +12406,7 @@ void rocksdb_set_update_cf_options(THD *const /* unused */, // the CF options. This will results in consistent behavior and avoids // dealing with cases when only a subset of CF-s was successfully updated. if (val) { + my_free(*reinterpret_cast(var_ptr)); *reinterpret_cast(var_ptr) = my_strdup(val, MYF(0)); } else { *reinterpret_cast(var_ptr) = nullptr; From 0acac4fe5fd6d8fab9f13859219f46842153803b Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 18 Dec 2017 01:55:40 +0400 Subject: [PATCH 07/23] MDEV-14593 human-readable XA RECOVER. The 'data' field in the XA RECOVER resultset changed to be charset_bin. It seems to me right and also --binary-as-hex starts working. The XA RECOVER FORMAT='SQL' option implemented. It returns the XID string that fits to be an argument for the XA ... statements. --- mysql-test/r/binary_to_hex.result | 4 + mysql-test/r/xa.result | 53 +++++++++++ mysql-test/t/binary_to_hex.test | 20 +++++ mysql-test/t/xa.test | 48 +++++++++- sql/handler.cc | 140 ++++++++++++++++++++++++++++-- sql/handler.h | 9 ++ sql/share/errmsg-utf8.txt | 4 +- sql/sql_yacc.yy | 21 ++++- 8 files changed, 286 insertions(+), 13 deletions(-) diff --git a/mysql-test/r/binary_to_hex.result b/mysql-test/r/binary_to_hex.result index 345df1de538..51ee5fa1c62 100644 --- a/mysql-test/r/binary_to_hex.result +++ b/mysql-test/r/binary_to_hex.result @@ -115,3 +115,7 @@ c9: 0x000000000101000000000000000000F03F000000000000F03F #Print the table contents in html format
c1c2c3c4c5c6c7c8c9
0x74696E79626C6F622D74657874207265616461626C650x626C6F622D74657874207265616461626C650x6D656469756D626C6F622D74657874207265616461626C650x6C6F6E67626C6F622D74657874207265616461626C650x74657874207265616461626C650x010x630x7661726961626C650x000000000101000000000000000000F03F000000000000F03F
idcol1col2
10xAB1234000000000000000x123ABC
20xDE1234000000000000000x123DEF
DROP TABLE t1, t2; +create table t1 (a int); +formatID gtrid_length bqual_length data +1 3 2 0x7472316271 +DROP TABLE t1; diff --git a/mysql-test/r/xa.result b/mysql-test/r/xa.result index fdcf25f3a12..6c1ff867f89 100644 --- a/mysql-test/r/xa.result +++ b/mysql-test/r/xa.result @@ -60,6 +60,59 @@ a 20 disconnect con1; connection default; +xa start 'tr1'; +insert t1 values (40); +xa end 'tr1'; +xa prepare 'tr1'; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +1 3 0 'tr1' +xa rollback 'tr1'; +xa start 'tr1', 'bq'; +insert t1 values (40); +xa end 'tr1', 'bq'; +xa prepare 'tr1', 'bq'; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +1 3 2 'tr1','bq' +xa rollback 'tr1', 'bq'; +xa start 'tr1', 'bq', 3; +insert t1 values (40); +xa end 'tr1', 'bq', 3; +xa prepare 'tr1', 'bq', 3; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +3 3 2 'tr1','bq',3 +xa rollback 'tr1', 'bq', 3; +xa start 'tr1#$'; +insert t1 values (40); +xa end 'tr1#$'; +xa prepare 'tr1#$'; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +1 5 0 X'7472312324' +xa rollback 'tr1#$'; +xa start 'tr1#$', 'bq'; +insert t1 values (40); +xa end 'tr1#$', 'bq'; +xa prepare 'tr1#$', 'bq'; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +1 5 2 X'7472312324',X'6271' +xa rollback 'tr1#$', 'bq'; +xa start 'tr1#$', 'bq', 3; +insert t1 values (40); +xa end 'tr1#$', 'bq', 3; +xa prepare 'tr1#$', 'bq', 3; +xa recover format='RAW'; +formatID gtrid_length bqual_length data +3 5 2 tr1#$bq +xa recover format='PLAIN'; +ERROR HY000: Unknown XA RECOVER format name: 'PLAIN' +xa recover format='SQL'; +formatID gtrid_length bqual_length data +3 5 2 X'7472312324',X'6271',3 +xa rollback 'tr1#$', 'bq', 3; drop table t1; drop table if exists t1; create table t1(a int, b int, c varchar(20), primary key(a)) engine = innodb; diff --git a/mysql-test/t/binary_to_hex.test b/mysql-test/t/binary_to_hex.test index 8312a246d0c..be4fb301e40 100644 --- a/mysql-test/t/binary_to_hex.test +++ b/mysql-test/t/binary_to_hex.test @@ -72,5 +72,25 @@ SELECT * FROM t2; #Cleanup DROP TABLE t1, t2; +# MDEV-14593 human-readable XA RECOVER + +create table t1 (a int); + +--write_file $MYSQLTEST_VARDIR/tmp/mdev-14593.sql +DELIMITER / +XA START 'tr1', 'bq'/ +INSERT INTO t1 VALUES (0)/ +XA END 'tr1', 'bq'/ +XA PREPARE 'tr1', 'bq'/ +XA RECOVER/ +XA ROLLBACK 'tr1', 'bq'/ +EOF +--exec $MYSQL test --binary-as-hex < $MYSQLTEST_VARDIR/tmp/mdev-14593.sql 2>&1 +remove_file $MYSQLTEST_VARDIR/tmp/mdev-14593.sql; + + +#Cleanup +DROP TABLE t1; + # Wait till all disconnects are completed --source include/wait_until_count_sessions.inc diff --git a/mysql-test/t/xa.test b/mysql-test/t/xa.test index c1f36129d75..7a6208f4b38 100644 --- a/mysql-test/t/xa.test +++ b/mysql-test/t/xa.test @@ -83,6 +83,53 @@ select * from t1; disconnect con1; connection default; + +# MDEV-14593 human-readable XA RECOVER +xa start 'tr1'; +insert t1 values (40); +xa end 'tr1'; +xa prepare 'tr1'; +xa recover format='SQL'; +xa rollback 'tr1'; + +xa start 'tr1', 'bq'; +insert t1 values (40); +xa end 'tr1', 'bq'; +xa prepare 'tr1', 'bq'; +xa recover format='SQL'; +xa rollback 'tr1', 'bq'; + +xa start 'tr1', 'bq', 3; +insert t1 values (40); +xa end 'tr1', 'bq', 3; +xa prepare 'tr1', 'bq', 3; +xa recover format='SQL'; +xa rollback 'tr1', 'bq', 3; + +xa start 'tr1#$'; +insert t1 values (40); +xa end 'tr1#$'; +xa prepare 'tr1#$'; +xa recover format='SQL'; +xa rollback 'tr1#$'; + +xa start 'tr1#$', 'bq'; +insert t1 values (40); +xa end 'tr1#$', 'bq'; +xa prepare 'tr1#$', 'bq'; +xa recover format='SQL'; +xa rollback 'tr1#$', 'bq'; + +xa start 'tr1#$', 'bq', 3; +insert t1 values (40); +xa end 'tr1#$', 'bq', 3; +xa prepare 'tr1#$', 'bq', 3; +xa recover format='RAW'; +--error ER_UNKNOWN_EXPLAIN_FORMAT +xa recover format='PLAIN'; +xa recover format='SQL'; +xa rollback 'tr1#$', 'bq', 3; + drop table t1; # @@ -379,7 +426,6 @@ connection default; DROP TABLE t1, t2; disconnect con2; - # Wait till all disconnects are completed --source include/wait_until_count_sessions.inc diff --git a/sql/handler.cc b/sql/handler.cc index 81c6b60d256..93db0b09abf 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1981,6 +1981,97 @@ int ha_recover(HASH *commit_list) DBUG_RETURN(0); } +/** + return the XID as it appears in the SQL function's arguments. + So this string can be passed to XA START, XA PREPARE etc... + + @note + the 'buf' has to have space for at least SQL_XIDSIZE bytes. +*/ + + +/* + 'a'..'z' 'A'..'Z', '0'..'9' + and '-' '_' ' ' symbols don't have to be + converted. +*/ + +static const char xid_needs_conv[128]= +{ + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1, + 0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1, + 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0, + 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1 +}; + +uint get_sql_xid(XID *xid, char *buf) +{ + int tot_len= xid->gtrid_length + xid->bqual_length; + int i; + const char *orig_buf= buf; + + for (i=0; idata)[i]; + if (c >= 128 || xid_needs_conv[c]) + break; + } + + if (i >= tot_len) + { + /* No need to convert characters to hexadecimals. */ + *buf++= '\''; + memcpy(buf, xid->data, xid->gtrid_length); + buf+= xid->gtrid_length; + *buf++= '\''; + if (xid->bqual_length > 0 || xid->formatID != 1) + { + *buf++= ','; + *buf++= '\''; + memcpy(buf, xid->data+xid->gtrid_length, xid->bqual_length); + buf+= xid->bqual_length; + *buf++= '\''; + } + } + else + { + *buf++= 'X'; + *buf++= '\''; + for (i= 0; i < xid->gtrid_length; i++) + { + *buf++=_dig_vec_lower[((uchar*) xid->data)[i] >> 4]; + *buf++=_dig_vec_lower[((uchar*) xid->data)[i] & 0x0f]; + } + *buf++= '\''; + if (xid->bqual_length > 0 || xid->formatID != 1) + { + *buf++= ','; + *buf++= 'X'; + *buf++= '\''; + for (; i < tot_len; i++) + { + *buf++=_dig_vec_lower[((uchar*) xid->data)[i] >> 4]; + *buf++=_dig_vec_lower[((uchar*) xid->data)[i] & 0x0f]; + } + *buf++= '\''; + } + } + + if (xid->formatID != 1) + { + *buf++= ','; + buf+= my_longlong10_to_str_8bit(&my_charset_bin, buf, + MY_INT64_NUM_DECIMAL_DIGITS, -10, xid->formatID); + } + + return buf - orig_buf; +} + + /** return the list of XID's to a client, the same way SHOW commands do. @@ -1990,7 +2081,8 @@ int ha_recover(HASH *commit_list) It can be easily fixed later, if necessary. */ -static my_bool xa_recover_callback(XID_STATE *xs, Protocol *protocol) +static my_bool xa_recover_callback(XID_STATE *xs, Protocol *protocol, + char *data, uint data_len, CHARSET_INFO *data_cs) { if (xs->xa_state == XA_PREPARED) { @@ -1998,8 +2090,7 @@ static my_bool xa_recover_callback(XID_STATE *xs, Protocol *protocol) protocol->store_longlong((longlong) xs->xid.formatID, FALSE); protocol->store_longlong((longlong) xs->xid.gtrid_length, FALSE); protocol->store_longlong((longlong) xs->xid.bqual_length, FALSE); - protocol->store(xs->xid.data, xs->xid.gtrid_length + xs->xid.bqual_length, - &my_charset_bin); + protocol->store(data, data_len, data_cs); if (protocol->write()) return TRUE; } @@ -2007,11 +2098,28 @@ static my_bool xa_recover_callback(XID_STATE *xs, Protocol *protocol) } +static my_bool xa_recover_callback_short(XID_STATE *xs, Protocol *protocol) +{ + return xa_recover_callback(xs, protocol, xs->xid.data, + xs->xid.gtrid_length + xs->xid.bqual_length, &my_charset_bin); +} + + +static my_bool xa_recover_callback_verbose(XID_STATE *xs, Protocol *protocol) +{ + char buf[SQL_XIDSIZE]; + uint len= get_sql_xid(&xs->xid, buf); + return xa_recover_callback(xs, protocol, buf, len, + &my_charset_utf8_general_ci); +} + + bool mysql_xa_recover(THD *thd) { List field_list; Protocol *protocol= thd->protocol; MEM_ROOT *mem_root= thd->mem_root; + my_hash_walk_action action; DBUG_ENTER("mysql_xa_recover"); field_list.push_back(new (mem_root) @@ -2023,16 +2131,32 @@ bool mysql_xa_recover(THD *thd) field_list.push_back(new (mem_root) Item_int(thd, "bqual_length", 0, MY_INT32_NUM_DECIMAL_DIGITS), mem_root); - field_list.push_back(new (mem_root) - Item_empty_string(thd, "data", - XIDDATASIZE), mem_root); + { + uint len; + CHARSET_INFO *cs; + + if (thd->lex->verbose) + { + len= SQL_XIDSIZE; + cs= &my_charset_utf8_general_ci; + action= (my_hash_walk_action) xa_recover_callback_verbose; + } + else + { + len= XIDDATASIZE; + cs= &my_charset_bin; + action= (my_hash_walk_action) xa_recover_callback_short; + } + + field_list.push_back(new (mem_root) + Item_empty_string(thd, "data", len, cs), mem_root); + } if (protocol->send_result_set_metadata(&field_list, Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF)) DBUG_RETURN(1); - if (xid_cache_iterate(thd, (my_hash_walk_action) xa_recover_callback, - protocol)) + if (xid_cache_iterate(thd, action, protocol)) DBUG_RETURN(1); my_eof(thd); DBUG_RETURN(0); diff --git a/sql/handler.h b/sql/handler.h index 9ada2ccbe7c..9941462da8d 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -652,6 +652,15 @@ struct xid_t { }; typedef struct xid_t XID; +/* + The size of XID string representation in the form + 'gtrid', 'bqual', formatID + see xid_t::get_sql_string() for details. +*/ +#define SQL_XIDSIZE (XIDDATASIZE * 2 + 8 + MY_INT64_NUM_DECIMAL_DIGITS) +/* The 'buf' has to have space for at least SQL_XIDSIZE bytes. */ +uint get_sql_xid(XID *xid, char *buf); + /* for recover() handlerton call */ #define MIN_XID_LIST_SIZE 128 #define MAX_XID_LIST_SIZE (1024*128) diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index c55fe6686a4..5d457bc57a4 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -6850,8 +6850,8 @@ ER_CANT_SET_GTID_NEXT_WHEN_OWNING_GTID eng "GTID_NEXT cannot be changed by a client that owns a GTID. The client owns %s. Ownership is released on COMMIT or ROLLBACK" ER_UNKNOWN_EXPLAIN_FORMAT - eng "Unknown EXPLAIN format name: '%s'" - rus "Неизвестное имя формата команды EXPLAIN: '%s'" + eng "Unknown %s format name: '%s'" + rus "Неизвестное имя формата команды %s: '%s'" ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION 25006 eng "Cannot execute statement in a READ ONLY transaction" diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index e4661e8f9d1..05f0998683e 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1682,7 +1682,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); opt_default_time_precision case_stmt_body opt_bin_mod opt_if_exists_table_element opt_if_not_exists_table_element - opt_recursive + opt_recursive opt_format_xid %type create_or_replace @@ -16810,9 +16810,26 @@ xa: { Lex->sql_command = SQLCOM_XA_ROLLBACK; } - | XA_SYM RECOVER_SYM + | XA_SYM RECOVER_SYM opt_format_xid { Lex->sql_command = SQLCOM_XA_RECOVER; + Lex->verbose= $3; + } + ; + +opt_format_xid: + /* empty */ { $$= false; } + | FORMAT_SYM '=' ident_or_text + { + if (!my_strcasecmp(system_charset_info, $3.str, "SQL")) + $$= true; + else if (!my_strcasecmp(system_charset_info, $3.str, "RAW")) + $$= false; + else + { + my_yyabort_error((ER_UNKNOWN_EXPLAIN_FORMAT, MYF(0), "XA RECOVER", $3.str)); + $$= false; + } } ; From 7fd78055743ba54a3db45f4947674882bf3641e6 Mon Sep 17 00:00:00 2001 From: Vesa Pentti Date: Mon, 11 Dec 2017 19:20:37 +0000 Subject: [PATCH 08/23] MDEV-14315 -- Reflect use of tcmalloc in a system variable and error log * The version of tcmalloc is written to the system variable 'version_malloc_library' if tcmalloc is used, similarly to jemalloc * Extracted method guess_malloc_library() --- include/my_sys.h | 2 ++ mysys/CMakeLists.txt | 3 +- mysys/guess_malloc_library.c | 63 ++++++++++++++++++++++++++++++++++++ sql/sys_vars.cc | 22 +------------ 4 files changed, 68 insertions(+), 22 deletions(-) create mode 100644 mysys/guess_malloc_library.c diff --git a/include/my_sys.h b/include/my_sys.h index fb3e15b64a4..adb7f4826f6 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -154,6 +154,8 @@ typedef struct my_aio_result { /* Extra length needed for filename if one calls my_create_backup_name */ #define MY_BACKUP_NAME_EXTRA_LENGTH 17 +char *guess_malloc_library(); + /* If we have our own safemalloc (for debugging) */ #if defined(SAFEMALLOC) void sf_report_leaked_memory(my_thread_id id); diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index b9f92124cc3..8e4882328f5 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -36,7 +36,8 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c string.c thr_alarm.c thr_lock.c thr_mutex.c thr_rwlock.c thr_timer.c tree.c typelib.c base64.c my_memmem.c - my_getpagesize.c + my_getpagesize.c + guess_malloc_library.c lf_alloc-pin.c lf_dynarray.c lf_hash.c safemalloc.c my_new.cc my_atomic.c my_getncpus.c my_safehash.c my_chmod.c my_rnd.c diff --git a/mysys/guess_malloc_library.c b/mysys/guess_malloc_library.c new file mode 100644 index 00000000000..2e640757e11 --- /dev/null +++ b/mysys/guess_malloc_library.c @@ -0,0 +1,63 @@ +/* Copyright (c) 2002, 2015, Oracle and/or its affiliates. + Copyright (c) 2012, 2017, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/* guess_malloc_library() deduces, to the best of its ability, + the currently used malloc library and its version */ + +#include +#include + +char *guess_malloc_library() +{ +#ifndef HAVE_DLOPEN + return (char*) MALLOC_LIBRARY; +#else + static char buf[128]; + + if (strcmp(MALLOC_LIBRARY, "system") != 0) + { + return (char*) MALLOC_LIBRARY; + } + + /* tcmalloc */ + typedef const char* (*tc_version_type)(int*, int*, const char**); + tc_version_type tc_version_func = + (tc_version_type) dlsym(RTLD_DEFAULT, "tc_version"); + if (tc_version_func) + { + int major, minor; + const char* ver_str = tc_version_func(&major, &minor, NULL); + strxnmov(buf, sizeof(buf)-1, "tcmalloc ", ver_str, NULL); + return buf; + } + + /* jemalloc */ + typedef int (*mallctl_type)(const char*, void*, size_t*, void*, size_t); + mallctl_type mallctl_func = + (mallctl_type) dlsym(RTLD_DEFAULT, "mallctl"); + if (mallctl_func) + { + char *ver; + size_t len = sizeof(ver); + mallctl_func("version", &ver, &len, NULL, 0); + strxnmov(buf, sizeof(buf)-1, "jemalloc ", ver, NULL); + return buf; + } + + return (char*) MALLOC_LIBRARY; +#endif +} + diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 78242e0a088..a517ab0965e 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -35,6 +35,7 @@ #include "sql_priv.h" #include "sql_class.h" // set_var.h: THD #include "sys_vars.ic" +#include "my_sys.h" #include "events.h" #include @@ -3511,27 +3512,6 @@ static Sys_var_charptr Sys_version_source_revision( CMD_LINE_HELP_ONLY, IN_SYSTEM_CHARSET, DEFAULT(SOURCE_REVISION)); -static char *guess_malloc_library() -{ - if (strcmp(MALLOC_LIBRARY, "system") == 0) - { -#ifdef HAVE_DLOPEN - typedef int (*mallctl_type)(const char*, void*, size_t*, void*, size_t); - mallctl_type mallctl_func; - mallctl_func= (mallctl_type)dlsym(RTLD_DEFAULT, "mallctl"); - if (mallctl_func) - { - static char buf[128]; - char *ver; - size_t len = sizeof(ver); - mallctl_func("version", &ver, &len, NULL, 0); - strxnmov(buf, sizeof(buf)-1, "jemalloc ", ver, NULL); - return buf; - } -#endif - } - return const_cast(MALLOC_LIBRARY); -} static char *malloc_library; static Sys_var_charptr Sys_malloc_library( "version_malloc_library", "Version of the used malloc library", From c1e5fef05d87038fff8b55ba763d082a5bec5d78 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 18 Dec 2017 11:25:38 +0400 Subject: [PATCH 09/23] MDEV-14008 Assertion failing: `!is_set() || (m_status == DA_OK_BULK && is_bulk_op()) --- mysql-test/suite/innodb/r/innodb-autoinc.result | 12 ++++++++++++ mysql-test/suite/innodb/t/innodb-autoinc.test | 12 ++++++++++++ sql/field.cc | 4 ++-- sql/field.h | 8 +++++++- storage/innobase/handler/ha_innodb.cc | 4 ++-- storage/xtradb/handler/ha_innodb.cc | 4 ++-- 6 files changed, 37 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-autoinc.result b/mysql-test/suite/innodb/r/innodb-autoinc.result index 879f9dfa238..9aa819de22b 100644 --- a/mysql-test/suite/innodb/r/innodb-autoinc.result +++ b/mysql-test/suite/innodb/r/innodb-autoinc.result @@ -1348,3 +1348,15 @@ t CREATE TABLE `t` ( KEY `i` (`i`) ) ENGINE=InnoDB AUTO_INCREMENT=401 DEFAULT CHARSET=latin1 DROP TABLE t; +# +# MDEV-14008 Assertion failing: `!is_set() || (m_status == DA_OK_BULK && is_bulk_op()) +# +SET sql_mode=STRICT_ALL_TABLES; +CREATE TABLE t1 ( +c1 DOUBLE NOT NULL PRIMARY KEY AUTO_INCREMENT +) ENGINE=InnoDB AUTO_INCREMENT=10000000000000000000; +INSERT INTO t1 VALUES (); +SELECT * FROM t1; +c1 +1e19 +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-autoinc.test b/mysql-test/suite/innodb/t/innodb-autoinc.test index 362be2e055b..ebb6a5d24ff 100644 --- a/mysql-test/suite/innodb/t/innodb-autoinc.test +++ b/mysql-test/suite/innodb/t/innodb-autoinc.test @@ -680,3 +680,15 @@ INSERT INTO t VALUES (NULL); SELECT * FROM t; SHOW CREATE TABLE t; DROP TABLE t; + +--echo # +--echo # MDEV-14008 Assertion failing: `!is_set() || (m_status == DA_OK_BULK && is_bulk_op()) +--echo # + +SET sql_mode=STRICT_ALL_TABLES; +CREATE TABLE t1 ( + c1 DOUBLE NOT NULL PRIMARY KEY AUTO_INCREMENT +) ENGINE=InnoDB AUTO_INCREMENT=10000000000000000000; +INSERT INTO t1 VALUES (); +SELECT * FROM t1; +DROP TABLE t1; diff --git a/sql/field.cc b/sql/field.cc index 7074cc2cbc4..a3c20ec18f2 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -4390,7 +4390,7 @@ double Field_double::val_real(void) return j; } -longlong Field_double::val_int(void) +longlong Field_double::val_int_from_real(bool want_unsigned_result) { ASSERT_COLUMN_MARKED_FOR_READ; double j; @@ -4398,7 +4398,7 @@ longlong Field_double::val_int(void) bool error; float8get(j,ptr); - res= double_to_longlong(j, 0, &error); + res= double_to_longlong(j, want_unsigned_result, &error); if (error) { ErrConvDouble err(j); diff --git a/sql/field.h b/sql/field.h index e7bd5532ae6..c99327bd068 100644 --- a/sql/field.h +++ b/sql/field.h @@ -420,6 +420,10 @@ public: enum_check_fields check_level); virtual double val_real(void)=0; virtual longlong val_int(void)=0; + virtual ulonglong val_uint(void) + { + return (ulonglong) val_int(); + } virtual my_decimal *val_decimal(my_decimal *); inline String *val_str(String *str) { return val_str(str, str); } /* @@ -1554,6 +1558,7 @@ private: class Field_double :public Field_real { + longlong val_int_from_real(bool want_unsigned_result); public: Field_double(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, uchar null_bit_arg, @@ -1580,7 +1585,8 @@ public: int store(longlong nr, bool unsigned_val); int reset(void) { bzero(ptr,sizeof(double)); return 0; } double val_real(void); - longlong val_int(void); + longlong val_int(void) { return val_int_from_real(false); } + ulonglong val_uint(void) { return (ulonglong) val_int_from_real(true); } String *val_str(String*,String *); bool send_binary(Protocol *protocol); int cmp(const uchar *,const uchar *); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4aab1b8d713..59a8aedd266 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -7274,7 +7274,7 @@ no_commit: table->next_number_field); /* Get the value that MySQL attempted to store in the table.*/ - auto_inc = table->next_number_field->val_int(); + auto_inc = table->next_number_field->val_uint(); switch (error) { case DB_DUPLICATE_KEY: @@ -7735,7 +7735,7 @@ ha_innobase::update_row( ulonglong auto_inc; ulonglong col_max_value; - auto_inc = table->next_number_field->val_int(); + auto_inc = table->next_number_field->val_uint(); /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index f111a576d8f..88b59db9a46 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -7988,7 +7988,7 @@ no_commit: table->next_number_field); /* Get the value that MySQL attempted to store in the table.*/ - auto_inc = table->next_number_field->val_int(); + auto_inc = table->next_number_field->val_uint(); switch (error) { case DB_DUPLICATE_KEY: @@ -8468,7 +8468,7 @@ ha_innobase::update_row( ulonglong auto_inc; ulonglong col_max_value; - auto_inc = table->next_number_field->val_int(); + auto_inc = table->next_number_field->val_uint(); /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ From 8ed78cf7f93f6129c11ee979500971c11e15f6f9 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 18 Dec 2017 12:24:02 +0400 Subject: [PATCH 10/23] xa.test fixed to be thread-stable. --- mysql-test/t/xa.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/t/xa.test b/mysql-test/t/xa.test index 7a6208f4b38..b26f3f3c73f 100644 --- a/mysql-test/t/xa.test +++ b/mysql-test/t/xa.test @@ -80,8 +80,9 @@ xa rollback 'testa','testb'; xa start 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'; select * from t1; - disconnect con1; +--source include/wait_until_count_sessions.inc + connection default; # MDEV-14593 human-readable XA RECOVER From 6ee9cba74502e2da628ac72b21ce42570ff361b9 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 18 Dec 2017 15:21:50 +0400 Subject: [PATCH 11/23] MDEV-10486 MariaDB 10.x does not update rows_examined in performance_schema tables. Save the rows_examined counter before it gets emptied. --- mysql-test/suite/perfschema/r/misc.result | 16 ++++++++++++++++ mysql-test/suite/perfschema/t/misc.test | 15 +++++++++++++++ sql/sql_parse.cc | 5 +++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/perfschema/r/misc.result b/mysql-test/suite/perfschema/r/misc.result index 2adf2cba851..7a097a27576 100644 --- a/mysql-test/suite/perfschema/r/misc.result +++ b/mysql-test/suite/perfschema/r/misc.result @@ -118,3 +118,19 @@ B select count(*) from events_statements_history where sql_text like "%..."; count(*) 2 +use test; +create table t1 (id int); +insert into t1 values (1), (2), (3); +truncate performance_schema.events_statements_history; +select * from t1; +id +1 +2 +3 +insert into t1 select RAND()*10000 from t1; +select sql_text, rows_examined from performance_schema.events_statements_history; +sql_text rows_examined +truncate performance_schema.events_statements_history 0 +select * from t1 3 +insert into t1 select RAND()*10000 from t1 6 +drop table t1; diff --git a/mysql-test/suite/perfschema/t/misc.test b/mysql-test/suite/perfschema/t/misc.test index bf3e8afffdc..c9f7dc6bfc0 100644 --- a/mysql-test/suite/perfschema/t/misc.test +++ b/mysql-test/suite/perfschema/t/misc.test @@ -207,3 +207,18 @@ select 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa select _utf8mb4 'васÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑ' as B; select count(*) from events_statements_history where sql_text like "%..."; + + +# +# MDEV-10486 MariaDB 10.x does not update rows_examined in performance_schema tables +# Verify that the rows_examined counter is set properly. + +use test; +create table t1 (id int); +insert into t1 values (1), (2), (3); +truncate performance_schema.events_statements_history; +select * from t1; +insert into t1 select RAND()*10000 from t1; +select sql_text, rows_examined from performance_schema.events_statements_history; +drop table t1; + diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 5cea264e4a8..3121ee99834 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1957,11 +1957,12 @@ bool dispatch_command(enum enum_server_command command, THD *thd, THD_STAGE_INFO(thd, stage_cleaning_up); thd->reset_query(); - thd->set_examined_row_count(0); // For processlist - thd->set_command(COM_SLEEP); /* Performance Schema Interface instrumentation, end */ MYSQL_END_STATEMENT(thd->m_statement_psi, thd->get_stmt_da()); + thd->set_examined_row_count(0); // For processlist + thd->set_command(COM_SLEEP); + thd->m_statement_psi= NULL; thd->m_digest= NULL; From 03e91ce324465e465468021edd050c3b73d4cee3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 15 Dec 2017 16:38:46 +0100 Subject: [PATCH 12/23] MDEV-14641 Incompatible key or row definition between the MariaDB .frm file and the information in the storage engine make sure that mysql_create_frm_image() and fast_alter_partition_table() use the same code to derive HA_OPTION_PACK_RECORD from create_info->row_type. --- .../suite/parts/r/partition_alter_myisam.result | 10 ++++++++++ .../suite/parts/t/partition_alter_myisam.test | 17 +++++++++++++++++ sql/handler.h | 7 +++++++ sql/sql_partition.cc | 5 +---- sql/sql_table.cc | 5 +---- 5 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/parts/r/partition_alter_myisam.result create mode 100644 mysql-test/suite/parts/t/partition_alter_myisam.test diff --git a/mysql-test/suite/parts/r/partition_alter_myisam.result b/mysql-test/suite/parts/r/partition_alter_myisam.result new file mode 100644 index 00000000000..514593fd4ef --- /dev/null +++ b/mysql-test/suite/parts/r/partition_alter_myisam.result @@ -0,0 +1,10 @@ +CREATE TABLE t1 (i INT) ENGINE=MYISAM +PARTITION BY LIST(i) ( +PARTITION p0 VALUES IN (1), +PARTITION p1 VALUES IN (2) +); +ALTER TABLE t1 ROW_FORMAT=COMPRESSED; +ALTER TABLE t1 DROP PARTITION p1; +SELECT * FROM t1; +i +DROP TABLE t1; diff --git a/mysql-test/suite/parts/t/partition_alter_myisam.test b/mysql-test/suite/parts/t/partition_alter_myisam.test new file mode 100644 index 00000000000..91ce8d21327 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_alter_myisam.test @@ -0,0 +1,17 @@ +# +# MDEV-14641 Incompatible key or row definition between the MariaDB .frm file and the information in the storage engine +# + +--source include/have_partition.inc + +CREATE TABLE t1 (i INT) ENGINE=MYISAM +PARTITION BY LIST(i) ( + PARTITION p0 VALUES IN (1), + PARTITION p1 VALUES IN (2) +); +ALTER TABLE t1 ROW_FORMAT=COMPRESSED; +ALTER TABLE t1 DROP PARTITION p1; +SELECT * FROM t1; + +# Cleanup +DROP TABLE t1; diff --git a/sql/handler.h b/sql/handler.h index 772f2e68dab..c422094b4d5 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1666,6 +1666,13 @@ struct HA_CREATE_INFO used_fields|= (HA_CREATE_USED_CHARSET | HA_CREATE_USED_DEFAULT_CHARSET); return false; } + ulong table_options_with_row_type() + { + if (row_type == ROW_TYPE_DYNAMIC || row_type == ROW_TYPE_PAGE) + return table_options | HA_OPTION_PACK_RECORD; + else + return table_options; + } }; diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index 02109b22898..a675d325a8b 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -6834,10 +6834,7 @@ uint fast_alter_partition_table(THD *thd, TABLE *table, lpt->part_info= part_info; lpt->alter_info= alter_info; lpt->create_info= create_info; - lpt->db_options= create_info->table_options; - if (create_info->row_type != ROW_TYPE_FIXED && - create_info->row_type != ROW_TYPE_DEFAULT) - lpt->db_options|= HA_OPTION_PACK_RECORD; + lpt->db_options= create_info->table_options_with_row_type(); lpt->table= table; lpt->key_info_buffer= 0; lpt->key_count= 0; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 9549884bc4e..7d37c559e20 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4405,10 +4405,7 @@ handler *mysql_create_frm_image(THD *thd, set_table_default_charset(thd, create_info, (char*) db); - db_options= create_info->table_options; - if (create_info->row_type == ROW_TYPE_DYNAMIC || - create_info->row_type == ROW_TYPE_PAGE) - db_options|= HA_OPTION_PACK_RECORD; + db_options= create_info->table_options_with_row_type(); if (!(file= get_new_handler((TABLE_SHARE*) 0, thd->mem_root, create_info->db_type))) From f32063c5137adc028a9b038b9e99f9384e583d0b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 18 Dec 2017 15:37:06 +0000 Subject: [PATCH 13/23] MDEV-13620 - improve help message for 'plugin-dir' and 'plugin-load' options. --- extra/mariabackup/xtrabackup.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 4d179961d1d..eb1cbf4748d 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1103,11 +1103,13 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &defaults_group, (G_PTR*) &defaults_group, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"plugin-dir", OPT_PLUGIN_DIR, "Server plugin directory", + {"plugin-dir", OPT_PLUGIN_DIR, + "Server plugin directory. Used to load encryption plugin during 'prepare' phase." + "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", &xb_plugin_dir, &xb_plugin_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - { "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load", + { "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load during 'prepare' phase.", &xb_plugin_load, &xb_plugin_load, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, From 1c2f59f7fbacc0fa7c4037bf03630f849859ad45 Mon Sep 17 00:00:00 2001 From: Martynas Bendorius Date: Sun, 1 Oct 2017 15:45:51 +0200 Subject: [PATCH 14/23] MDEV-13969 sst mysqldump and xtrabackup-v2 handle WSREP_SST_OPT_CONF incorrectly wrep_sst_common: Setting "-c ''" for my_print_defaults just takes no values from config at all. $MY_PRINT_DEFAULTS is already set at the top of the script to have --defaults-file and --defaults-extra-file. If WSREP_SST_OPT_CONF if set to "--defaults-file=/etc/my.cnf --defaults-extra-file=/etc/my.extra.cnf", then "my_print_defaults -c "" --defaults-file=/etc/my.cnf" succeeds, but if WSREP_SST_OPT_CONF is empty - no default values are taken at all. wsrep_sst_xtrabackup-v2: innobackupex does not support --defaults-extra-file, so ${WSREP_SST_OPT_CONF} cannot be used as an argument, it has been changed to ${WSREP_SST_OPT_DEFAULT}. Removed --defaults-file= from INNOMOVE line, because WSREP_SST_OPT_CONF already includes it (INNOBACKUP was fine, INNOMOVE - not). --- scripts/wsrep_sst_common.sh | 4 ++-- scripts/wsrep_sst_xtrabackup-v2.sh | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 1ef4830661b..788e1a12f37 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -257,12 +257,12 @@ parse_cnf() # look in group+suffix if [ -n $WSREP_SST_OPT_CONF_SUFFIX ]; then - reval=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF "${group}${WSREP_SST_OPT_CONF_SUFFIX}" | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) + reval=$($MY_PRINT_DEFAULTS "${group}${WSREP_SST_OPT_CONF_SUFFIX}" | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) fi # look in group if [ -z $reval ]; then - reval=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) + reval=$($MY_PRINT_DEFAULTS $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) fi # use default if we haven't found a value diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index faa7bc5e815..40e686d4d6b 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -492,7 +492,7 @@ read_cnf() ssystag+="-" if [[ $ssyslog -ne -1 ]];then - if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then + if $MY_PRINT_DEFAULTS mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then ssyslog=1 fi fi @@ -669,7 +669,7 @@ check_extra() local use_socket=1 if [[ $uextra -eq 1 ]];then if $MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then - local eport=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + local eport=$($MY_PRINT_DEFAULTS mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) if [[ -n $eport ]];then # Xtrabackup works only locally. # Hence, setting host to 127.0.0.1 unconditionally. @@ -865,14 +865,14 @@ if [[ $ssyslog -eq 1 ]];then } INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " - INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " - INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" + INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_DEFAULT} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " + INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_DEFAULT} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" fi else INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" - INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" - INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" + INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_DEFAULT} --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" + INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_DEFAULT} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" fi get_stream From beabe6b2167ee7e8c2659da9507e33ddd56a9c29 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 16 Oct 2017 19:33:06 +0200 Subject: [PATCH 15/23] MDEV-13969 sst mysqldump and xtrabackup-v2 handle WSREP_SST_OPT_CONF incorrectly $WSREP_SST_OPT_CONF already includes --defaults-extra-file= prefix. --- scripts/wsrep_sst_mysqldump.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 59d9a3c5c9c..f086d9873d0 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -117,7 +117,7 @@ GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" |\ $MYSQL_CLIENT $AUTH -S$WSREP_SST_OPT_SOCKET --disable-reconnect --connect_timeout=10 |\ tail -1 | awk -F ' ' '{ print $2 }') -MYSQL="$MYSQL_CLIENT --defaults-extra-file=$WSREP_SST_OPT_CONF "\ +MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\ "$AUTH -h${WSREP_SST_OPT_HOST_UNESCAPED:-$WSREP_SST_OPT_HOST} "\ "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10" From 91daf8819cf84828b4c96be6f0b35caca97bb61c Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 12 Dec 2017 17:47:06 +0100 Subject: [PATCH 16/23] MW-416 Moved TOI replication to happen after ACL checking for commands: SQLCOM_CREATE_EVENT SQLCOM_ALTER_EVENT SQLCOM_DROP_EVENT SQLCOM_CREATE_VIEW SQLCOM_CREATE_TRIGGER SQLCOM_DROP_TRIGGER SQLCOM_INSTALL_PLUGIN SQLCOM_UNINSTALL_PLUGIN --- sql/events.cc | 18 ++++++++++++++++++ sql/sql_parse.cc | 7 ------- sql/sql_plugin.cc | 20 ++++++++++++++++---- sql/sql_trigger.cc | 5 +++++ sql/sql_view.cc | 5 +++++ 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/sql/events.cc b/sql/events.cc index 51f68ca4c03..cdc15f41716 100644 --- a/sql/events.cc +++ b/sql/events.cc @@ -335,6 +335,7 @@ Events::create_event(THD *thd, Event_parse_data *parse_data) if (check_access(thd, EVENT_ACL, parse_data->dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (lock_object_name(thd, MDL_key::EVENT, parse_data->dbname.str, parse_data->name.str)) @@ -417,6 +418,10 @@ Events::create_event(THD *thd, Event_parse_data *parse_data) thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); +#ifdef WITH_WSREP + error: + DBUG_RETURN(true); +#endif /* WITH_WSREP */ } @@ -457,6 +462,9 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, if (check_access(thd, EVENT_ACL, parse_data->dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + if (lock_object_name(thd, MDL_key::EVENT, parse_data->dbname.str, parse_data->name.str)) DBUG_RETURN(TRUE); @@ -541,6 +549,10 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); +#ifdef WITH_WSREP +error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -581,6 +593,8 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) if (check_access(thd, EVENT_ACL, dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + /* Turn off row binlogging of this statement and use statement-based so that all supporting tables are updated for DROP EVENT command. @@ -602,6 +616,10 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); +#ifdef WITH_WSREP +error: + DBUG_RETURN(TRUE); +#endif } diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3121ee99834..6084c59a257 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4565,7 +4565,6 @@ end_with_restore_list: if (res) break; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) switch (lex->sql_command) { case SQLCOM_CREATE_EVENT: { @@ -4599,7 +4598,6 @@ end_with_restore_list: lex->spname->m_name); break; case SQLCOM_DROP_EVENT: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (!(res= Events::drop_event(thd, lex->spname->m_db, lex->spname->m_name, lex->if_exists()))) @@ -5505,7 +5503,6 @@ end_with_restore_list: Note: SQLCOM_CREATE_VIEW also handles 'ALTER VIEW' commands as specified through the thd->lex->create_view_mode flag. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_view(thd, first_table, thd->lex->create_view_mode); break; } @@ -5521,7 +5518,6 @@ end_with_restore_list: case SQLCOM_CREATE_TRIGGER: { /* Conditionally writes to binlog. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_or_drop_trigger(thd, all_tables, 1); break; @@ -5529,7 +5525,6 @@ end_with_restore_list: case SQLCOM_DROP_TRIGGER: { /* Conditionally writes to binlog. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_or_drop_trigger(thd, all_tables, 0); break; } @@ -5594,13 +5589,11 @@ end_with_restore_list: my_ok(thd); break; case SQLCOM_INSTALL_PLUGIN: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (! (res= mysql_install_plugin(thd, &thd->lex->comment, &thd->lex->ident))) my_ok(thd); break; case SQLCOM_UNINSTALL_PLUGIN: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (! (res= mysql_uninstall_plugin(thd, &thd->lex->comment, &thd->lex->ident))) my_ok(thd); diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index f41d1e0fdbf..3a4097734ec 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2108,12 +2108,16 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, bool error; int argc=orig_argc; char **argv=orig_argv; + unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = + { MYSQL_AUDIT_GENERAL_CLASSMASK }; DBUG_ENTER("mysql_install_plugin"); tables.init_one_table("mysql", 5, "plugin", 6, "plugin", TL_WRITE); if (!opt_noacl && check_table_access(thd, INSERT_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table = open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT))) @@ -2146,8 +2150,6 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, See also mysql_uninstall_plugin() and initialize_audit_plugin() */ - unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = - { MYSQL_AUDIT_GENERAL_CLASSMASK }; mysql_audit_acquire_plugins(thd, event_class_mask); mysql_mutex_lock(&LOCK_plugin); @@ -2178,6 +2180,10 @@ err: if (argv) free_defaults(argv); DBUG_RETURN(error); +#ifdef WITH_WSREP +error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -2244,6 +2250,8 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, TABLE_LIST tables; LEX_STRING dl= *dl_arg; bool error= false; + unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = + { MYSQL_AUDIT_GENERAL_CLASSMASK }; DBUG_ENTER("mysql_uninstall_plugin"); tables.init_one_table("mysql", 5, "plugin", 6, "plugin", TL_WRITE); @@ -2251,6 +2259,8 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, if (!opt_noacl && check_table_access(thd, DELETE_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table= open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT))) DBUG_RETURN(TRUE); @@ -2276,8 +2286,6 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, See also mysql_install_plugin() and initialize_audit_plugin() */ - unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = - { MYSQL_AUDIT_GENERAL_CLASSMASK }; mysql_audit_acquire_plugins(thd, event_class_mask); mysql_mutex_lock(&LOCK_plugin); @@ -2307,6 +2315,10 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, mysql_mutex_unlock(&LOCK_plugin); DBUG_RETURN(error); +#ifdef WITH_WSREP +error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index 28e59319a50..0b4978b2862 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -441,6 +441,7 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create) my_error(ER_BINLOG_CREATE_ROUTINE_NEED_SUPER, MYF(0)); DBUG_RETURN(TRUE); } + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (!create) { @@ -606,6 +607,10 @@ end: my_ok(thd); DBUG_RETURN(result); +#ifdef WITH_WSREP + error: + DBUG_RETURN(true); +#endif /* WITH_WSREP */ } /** diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 90c94e6a503..629dd865b0e 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -425,6 +425,7 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, if ((res= create_view_precheck(thd, tables, view, mode))) goto err; + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; @@ -695,6 +696,10 @@ err: lex->link_first_table_back(view, link_to_local); unit->cleanup(); DBUG_RETURN(res || thd->is_error()); +#ifdef WITH_WSREP + error: + DBUG_RETURN(true); +#endif /* WITH_WSREP */ } From 682c3bfd259b3184232e5329558094faa0bc8562 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 21 Nov 2016 16:20:10 -0500 Subject: [PATCH 17/23] MDEV-10442: "Address already in use" on restart SST processes should inherit mysqld's process group. --- sql/wsrep_utils.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/wsrep_utils.cc b/sql/wsrep_utils.cc index 580c8bbd55c..8a72d754a43 100644 --- a/sql/wsrep_utils.cc +++ b/sql/wsrep_utils.cc @@ -264,7 +264,6 @@ process::process (const char* cmd, const char* type, char** env) err_ = posix_spawnattr_setflags (&attr, POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK | - /* start a new process group */ POSIX_SPAWN_SETPGROUP | POSIX_SPAWN_USEVFORK); if (err_) { From 40088bfc7e100294d622ab32a6e7cbcb79c12eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 18 Dec 2017 19:46:23 +0200 Subject: [PATCH 18/23] MDEV-13407 innodb.drop_table_background failed in buildbot with "Tablespace for table exists" The InnoDB background DROP TABLE queue is something that we should really remove, but are unable to until we remove dict_operation_lock so that DDL and DML operations can be combined in a single transaction. Because the queue is not persistent, it is not crash-safe. In stable versions of MariaDB, we can only try harder to drop all enqueued tables before server shutdown. row_mysql_drop_t::table_id: Replaces table_name. row_drop_tables_for_mysql_in_background(): Do not remove the entry from the list as long as the table exists. In this way, the table should eventually be dropped. --- storage/innobase/row/row0mysql.cc | 142 +++++++++--------------------- storage/xtradb/row/row0mysql.cc | 142 +++++++++--------------------- 2 files changed, 80 insertions(+), 204 deletions(-) diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 74a17b9a1c3..4a39fc311b0 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -73,7 +73,7 @@ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ - char* table_name; /*!< table name */ + table_id_t table_id; /*!< table id */ UT_LIST_NODE_T(row_mysql_drop_t)row_mysql_drop_list; /*!< list chain node */ }; @@ -136,19 +136,6 @@ row_mysql_is_system_table( || 0 == strcmp(name + 6, "db")); } -/*********************************************************************//** -If a table is not yet in the drop list, adds the table to the list of tables -which the master thread drops in background. We need this on Unix because in -ALTER TABLE MySQL may call drop table even if the table has running queries on -it. Also, if there are running foreign key checks on the table, we drop the -table lazily. -@return TRUE if the table was not yet in the drop list, and was added there */ -static -ibool -row_add_table_to_background_drop_list( -/*==================================*/ - const char* name); /*!< in: table name */ - /*******************************************************************//** Delays an INSERT, DELETE or UPDATE operation if the purge is lagging. */ static @@ -2727,7 +2714,7 @@ loop: mutex_enter(&row_drop_list_mutex); ut_a(row_mysql_drop_list_inited); - +next: drop = UT_LIST_GET_FIRST(row_mysql_drop_list); n_tables = UT_LIST_GET_LEN(row_mysql_drop_list); @@ -2740,62 +2727,39 @@ loop: return(n_tables + n_tables_dropped); } - DBUG_EXECUTE_IF("row_drop_tables_in_background_sleep", - os_thread_sleep(5000000); - ); + table = dict_table_open_on_id(drop->table_id, FALSE, + DICT_TABLE_OP_NORMAL); - table = dict_table_open_on_name(drop->table_name, FALSE, FALSE, - DICT_ERR_IGNORE_NONE); - - if (table == NULL) { - /* If for some reason the table has already been dropped - through some other mechanism, do not try to drop it */ - - goto already_dropped; - } - - if (!table->to_be_dropped) { - /* There is a scenario: the old table is dropped - just after it's added into drop list, and new - table with the same name is created, then we try - to drop the new table in background. */ - dict_table_close(table, FALSE, FALSE); - - goto already_dropped; + if (!table) { + n_tables_dropped++; + mutex_enter(&row_drop_list_mutex); + UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); + MONITOR_DEC(MONITOR_BACKGROUND_DROP_TABLE); + ut_free(drop); + goto next; } ut_a(!table->can_be_evicted); + if (!table->to_be_dropped) { + dict_table_close(table, FALSE, FALSE); + + mutex_enter(&row_drop_list_mutex); + UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); + UT_LIST_ADD_LAST(row_mysql_drop_list, row_mysql_drop_list, + drop); + goto next; + } + dict_table_close(table, FALSE, FALSE); if (DB_SUCCESS != row_drop_table_for_mysql_in_background( - drop->table_name)) { + table->name)) { /* If the DROP fails for some table, we return, and let the main thread retry later */ - return(n_tables + n_tables_dropped); } - n_tables_dropped++; - -already_dropped: - mutex_enter(&row_drop_list_mutex); - - UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); - - MONITOR_DEC(MONITOR_BACKGROUND_DROP_TABLE); - - ut_print_timestamp(stderr); - fputs(" InnoDB: Dropped table ", stderr); - ut_print_name(stderr, NULL, TRUE, drop->table_name); - fputs(" in background drop queue.\n", stderr); - - mem_free(drop->table_name); - - mem_free(drop); - - mutex_exit(&row_drop_list_mutex); - goto loop; } @@ -2827,14 +2791,13 @@ which the master thread drops in background. We need this on Unix because in ALTER TABLE MySQL may call drop table even if the table has running queries on it. Also, if there are running foreign key checks on the table, we drop the table lazily. -@return TRUE if the table was not yet in the drop list, and was added there */ +@return whether background DROP TABLE was scheduled for the first time */ static -ibool -row_add_table_to_background_drop_list( -/*==================================*/ - const char* name) /*!< in: table name */ +bool +row_add_table_to_background_drop_list(table_id_t table_id) { row_mysql_drop_t* drop; + bool added = true; mutex_enter(&row_drop_list_mutex); @@ -2845,31 +2808,21 @@ row_add_table_to_background_drop_list( drop != NULL; drop = UT_LIST_GET_NEXT(row_mysql_drop_list, drop)) { - if (strcmp(drop->table_name, name) == 0) { - /* Already in the list */ - - mutex_exit(&row_drop_list_mutex); - - return(FALSE); + if (drop->table_id == table_id) { + added = false; + goto func_exit; } } - drop = static_cast( - mem_alloc(sizeof(row_mysql_drop_t))); - - drop->table_name = mem_strdup(name); + drop = static_cast(ut_malloc(sizeof *drop)); + drop->table_id = table_id; UT_LIST_ADD_LAST(row_mysql_drop_list, row_mysql_drop_list, drop); MONITOR_INC(MONITOR_BACKGROUND_DROP_TABLE); - - /* fputs("InnoDB: Adding table ", stderr); - ut_print_name(stderr, trx, TRUE, drop->table_name); - fputs(" to background drop list\n", stderr); */ - +func_exit: mutex_exit(&row_drop_list_mutex); - - return(TRUE); + return added; } /*********************************************************************//** @@ -4043,7 +3996,7 @@ row_drop_table_for_mysql( DBUG_EXECUTE_IF("row_drop_table_add_to_background", - row_add_table_to_background_drop_list(table->name); + row_add_table_to_background_drop_list(table->id); err = DB_SUCCESS; goto funct_exit; ); @@ -4055,33 +4008,22 @@ row_drop_table_for_mysql( checks take an IS or IX lock on the table. */ if (table->n_foreign_key_checks_running > 0) { - - const char* save_tablename = table->name; - ibool added; - - added = row_add_table_to_background_drop_list(save_tablename); - - if (added) { + if (row_add_table_to_background_drop_list(table->id)) { ut_print_timestamp(stderr); fputs(" InnoDB: You are trying to drop table ", stderr); - ut_print_name(stderr, trx, TRUE, save_tablename); + ut_print_name(stderr, trx, TRUE, table->name); fputs("\n" "InnoDB: though there is a" " foreign key check running on it.\n" "InnoDB: Adding the table to" " the background drop queue.\n", stderr); - - /* We return DB_SUCCESS to MySQL though the drop will - happen lazily later */ - - err = DB_SUCCESS; - } else { - /* The table is already in the background drop list */ - err = DB_ERROR; } + /* We return DB_SUCCESS to MySQL though the drop will + happen lazily later */ + err = DB_SUCCESS; goto funct_exit; } @@ -4103,11 +4045,7 @@ row_drop_table_for_mysql( lock_remove_all_on_table(table, TRUE); ut_a(table->n_rec_locks == 0); } else if (table->n_ref_count > 0 || table->n_rec_locks > 0) { - ibool added; - - added = row_add_table_to_background_drop_list(table->name); - - if (added) { + if (row_add_table_to_background_drop_list(table->id)) { ut_print_timestamp(stderr); fputs(" InnoDB: Warning: MySQL is" " trying to drop table ", stderr); diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index ebdee381713..8a6e2728342 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -72,7 +72,7 @@ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ - char* table_name; /*!< table name */ + table_id_t table_id; /*!< table id */ UT_LIST_NODE_T(row_mysql_drop_t)row_mysql_drop_list; /*!< list chain node */ }; @@ -135,19 +135,6 @@ row_mysql_is_system_table( || 0 == strcmp(name + 6, "db")); } -/*********************************************************************//** -If a table is not yet in the drop list, adds the table to the list of tables -which the master thread drops in background. We need this on Unix because in -ALTER TABLE MySQL may call drop table even if the table has running queries on -it. Also, if there are running foreign key checks on the table, we drop the -table lazily. -@return TRUE if the table was not yet in the drop list, and was added there */ -static -ibool -row_add_table_to_background_drop_list( -/*==================================*/ - const char* name); /*!< in: table name */ - /*******************************************************************//** Delays an INSERT, DELETE or UPDATE operation if the purge is lagging. */ static @@ -2739,7 +2726,7 @@ loop: mutex_enter(&row_drop_list_mutex); ut_a(row_mysql_drop_list_inited); - +next: drop = UT_LIST_GET_FIRST(row_mysql_drop_list); n_tables = UT_LIST_GET_LEN(row_mysql_drop_list); @@ -2752,62 +2739,39 @@ loop: return(n_tables + n_tables_dropped); } - DBUG_EXECUTE_IF("row_drop_tables_in_background_sleep", - os_thread_sleep(5000000); - ); + table = dict_table_open_on_id(drop->table_id, FALSE, + DICT_TABLE_OP_NORMAL); - table = dict_table_open_on_name(drop->table_name, FALSE, FALSE, - DICT_ERR_IGNORE_NONE); - - if (table == NULL) { - /* If for some reason the table has already been dropped - through some other mechanism, do not try to drop it */ - - goto already_dropped; - } - - if (!table->to_be_dropped) { - /* There is a scenario: the old table is dropped - just after it's added into drop list, and new - table with the same name is created, then we try - to drop the new table in background. */ - dict_table_close(table, FALSE, FALSE); - - goto already_dropped; + if (!table) { + n_tables_dropped++; + mutex_enter(&row_drop_list_mutex); + UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); + MONITOR_DEC(MONITOR_BACKGROUND_DROP_TABLE); + ut_free(drop); + goto next; } ut_a(!table->can_be_evicted); + if (!table->to_be_dropped) { + dict_table_close(table, FALSE, FALSE); + + mutex_enter(&row_drop_list_mutex); + UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); + UT_LIST_ADD_LAST(row_mysql_drop_list, row_mysql_drop_list, + drop); + goto next; + } + dict_table_close(table, FALSE, FALSE); if (DB_SUCCESS != row_drop_table_for_mysql_in_background( - drop->table_name)) { + table->name)) { /* If the DROP fails for some table, we return, and let the main thread retry later */ - return(n_tables + n_tables_dropped); } - n_tables_dropped++; - -already_dropped: - mutex_enter(&row_drop_list_mutex); - - UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); - - MONITOR_DEC(MONITOR_BACKGROUND_DROP_TABLE); - - ut_print_timestamp(stderr); - fputs(" InnoDB: Dropped table ", stderr); - ut_print_name(stderr, NULL, TRUE, drop->table_name); - fputs(" in background drop queue.\n", stderr); - - mem_free(drop->table_name); - - mem_free(drop); - - mutex_exit(&row_drop_list_mutex); - goto loop; } @@ -2839,14 +2803,13 @@ which the master thread drops in background. We need this on Unix because in ALTER TABLE MySQL may call drop table even if the table has running queries on it. Also, if there are running foreign key checks on the table, we drop the table lazily. -@return TRUE if the table was not yet in the drop list, and was added there */ +@return whether background DROP TABLE was scheduled for the first time */ static -ibool -row_add_table_to_background_drop_list( -/*==================================*/ - const char* name) /*!< in: table name */ +bool +row_add_table_to_background_drop_list(table_id_t table_id) { row_mysql_drop_t* drop; + bool added = true; mutex_enter(&row_drop_list_mutex); @@ -2857,31 +2820,21 @@ row_add_table_to_background_drop_list( drop != NULL; drop = UT_LIST_GET_NEXT(row_mysql_drop_list, drop)) { - if (strcmp(drop->table_name, name) == 0) { - /* Already in the list */ - - mutex_exit(&row_drop_list_mutex); - - return(FALSE); + if (drop->table_id == table_id) { + added = false; + goto func_exit; } } - drop = static_cast( - mem_alloc(sizeof(row_mysql_drop_t))); - - drop->table_name = mem_strdup(name); + drop = static_cast(ut_malloc(sizeof *drop)); + drop->table_id = table_id; UT_LIST_ADD_LAST(row_mysql_drop_list, row_mysql_drop_list, drop); MONITOR_INC(MONITOR_BACKGROUND_DROP_TABLE); - - /* fputs("InnoDB: Adding table ", stderr); - ut_print_name(stderr, trx, TRUE, drop->table_name); - fputs(" to background drop list\n", stderr); */ - +func_exit: mutex_exit(&row_drop_list_mutex); - - return(TRUE); + return added; } /*********************************************************************//** @@ -4057,7 +4010,7 @@ row_drop_table_for_mysql( DBUG_EXECUTE_IF("row_drop_table_add_to_background", - row_add_table_to_background_drop_list(table->name); + row_add_table_to_background_drop_list(table->id); err = DB_SUCCESS; goto funct_exit; ); @@ -4069,33 +4022,22 @@ row_drop_table_for_mysql( checks take an IS or IX lock on the table. */ if (table->n_foreign_key_checks_running > 0) { - - const char* save_tablename = table->name; - ibool added; - - added = row_add_table_to_background_drop_list(save_tablename); - - if (added) { + if (row_add_table_to_background_drop_list(table->id)) { ut_print_timestamp(stderr); fputs(" InnoDB: You are trying to drop table ", stderr); - ut_print_name(stderr, trx, TRUE, save_tablename); + ut_print_name(stderr, trx, TRUE, table->name); fputs("\n" "InnoDB: though there is a" " foreign key check running on it.\n" "InnoDB: Adding the table to" " the background drop queue.\n", stderr); - - /* We return DB_SUCCESS to MySQL though the drop will - happen lazily later */ - - err = DB_SUCCESS; - } else { - /* The table is already in the background drop list */ - err = DB_ERROR; } + /* We return DB_SUCCESS to MySQL though the drop will + happen lazily later */ + err = DB_SUCCESS; goto funct_exit; } @@ -4117,11 +4059,7 @@ row_drop_table_for_mysql( lock_remove_all_on_table(table, TRUE); ut_a(table->n_rec_locks == 0); } else if (table->n_ref_count > 0 || table->n_rec_locks > 0) { - ibool added; - - added = row_add_table_to_background_drop_list(table->name); - - if (added) { + if (row_add_table_to_background_drop_list(table->id)) { ut_print_timestamp(stderr); fputs(" InnoDB: Warning: MySQL is" " trying to drop table ", stderr); From 64f1fab068582a82ac54ce4793542655d5fc58ab Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 19 Dec 2017 10:24:50 +1100 Subject: [PATCH 19/23] MDEV-12128: systemd - add Documentation= directives --- support-files/mariadb.service.in | 4 +++- support-files/mariadb@.service.in | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index 6a307b2c41f..2b02a1d0db9 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -13,7 +13,9 @@ # and probably others [Unit] -Description=MariaDB database server +Description=MariaDB @VERSION@ database server +Documentation=man:mysqld(8) +Documentation=https://mariadb.com/kb/en/library/systemd/ After=network.target After=syslog.target diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index 410e7433b2b..a4a6a0ffef6 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -18,7 +18,9 @@ # Inspired from https://gitweb.gentoo.org/repo/gentoo.git/tree/dev-db/mysql-init-scripts/files/mysqld_at.service [Unit] -Description=MariaDB database server +Description=MariaDB @VERSION@ database server (multi-instance) +Documentation=man:mysqld(8) +Documentation=https://mariadb.com/kb/en/library/systemd/ After=network.target After=syslog.target From ce4cdfa0f8691f47366d50acd6049b8cd7520e8a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 19 Dec 2017 08:56:31 +1100 Subject: [PATCH 20/23] MDEV-13809: [service] should [Service] in systemd service files --- support-files/mariadb.service.in | 5 +++-- support-files/mariadb@.service.in | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index 2b02a1d0db9..fe00f160d28 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -78,7 +78,7 @@ ExecStartPre=/bin/sh -c "[ ! -e @bindir@/galera_recovery ] && VAR= || \ # Start main service # MYSQLD_OPTS here is for users to set in /etc/systemd/system/mariadb.service.d/MY_SPECIAL.conf -# Use the [service] section and Environment="MYSQLD_OPTS=...". +# Use the [Service] section and Environment="MYSQLD_OPTS=...". # This isn't a replacement for my.cnf. # _WSREP_NEW_CLUSTER is for the exclusive use of the script galera_new_cluster ExecStart=@sbindir@/mysqld $MYSQLD_OPTS $_WSREP_NEW_CLUSTER $_WSREP_START_POSITION @@ -105,7 +105,8 @@ UMask=007 ## ## ## by creating a file in /etc/systemd/system/mariadb.service.d/MY_SPECIAL.conf -## and adding/setting the following will override this file's settings. +## and adding/setting the following under [Service] will override this file's +## settings. # Useful options not previously available in [mysqld_safe] diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index a4a6a0ffef6..000724d7fe2 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -91,7 +91,7 @@ ExecStartPre=/bin/sh -c "[ ! -e @bindir@/galera_recovery ] && VAR= || \ # Start main service # MYSQLD_OPTS here is for users to set in /etc/systemd/system/mariadb@.service.d/MY_SPECIAL.conf -# Use the [service] section and Environment="MYSQLD_OPTS=...". +# Use the [Service] section and Environment="MYSQLD_OPTS=...". # This isn't a replacement for my.cnf. # _WSREP_NEW_CLUSTER is for the exclusive use of the script galera_new_cluster @@ -126,7 +126,8 @@ UMask=007 ## ## ## by creating a file in /etc/systemd/system/mariadb.service.d/MY_SPECIAL.conf -## and adding/setting the following will override this file's settings. +## and adding/setting the following below [Service] will override this file's +## settings. # Useful options not previously available in [mysqld_safe] From 079c3599711b9cbd3ce323f32cf99693cc3d5e3b Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Tue, 19 Dec 2017 11:49:40 +0200 Subject: [PATCH 21/23] MDEV-14629: failing assertion when a user-defined variable is defined by the recursive CTE During the user-defined variable defined by the recursive CTE handling procedure check_dependencies_in_with_clauses that checks dependencies between the tables that are defined in the CTE and find recursive definitions wasn't called. --- mysql-test/r/cte_recursive.result | 47 ++++++++++++++++++++++++++++ mysql-test/t/cte_recursive.test | 51 +++++++++++++++++++++++++++++++ sql/sql_parse.cc | 3 ++ 3 files changed, 101 insertions(+) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index b744f24bb57..ece5421e279 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2897,3 +2897,50 @@ n 1 2 3 +# +# mdev-14629: a user-defined variable is defined by the recursive CTE +# +set @var= +( +with recursive cte_tab(a) as ( +select 1 +union +select a+1 from cte_tab +where a<3) +select count(*) from cte_tab +); +select @var; +@var +3 +create table t1(a int, b int); +insert into t1 values (3,8),(1,5),(5,7),(7,4),(4,3); +set @var= +( +with recursive summ(a,s) as ( +select 1, 0 union +select t1.b, t1.b+summ.s from summ, t1 +where summ.a=t1.a) +select s from summ +order by a desc +limit 1 +); +select @var; +@var +27 +set @var= +( +with recursive +cte_1 as ( +select 1 +union +select * from cte_2), +cte_2 as ( +select * from cte_1 +union +select a from t1, cte_2 +where t1.a=cte_2.a) +select * from cte_2 +limit 1 +); +ERROR HY000: Unacceptable mutual recursion with anchored table 'cte_1' +drop table t1; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 36d948251c3..928f8a4334b 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1947,3 +1947,54 @@ cte2 as ( ) SELECT * FROM cte1; + +--echo # +--echo # mdev-14629: a user-defined variable is defined by the recursive CTE +--echo # + +set @var= +( + with recursive cte_tab(a) as ( + select 1 + union + select a+1 from cte_tab + where a<3) + select count(*) from cte_tab +); + +select @var; + +create table t1(a int, b int); +insert into t1 values (3,8),(1,5),(5,7),(7,4),(4,3); + +set @var= +( + with recursive summ(a,s) as ( + select 1, 0 union + select t1.b, t1.b+summ.s from summ, t1 + where summ.a=t1.a) + select s from summ + order by a desc + limit 1 +); + +select @var; + +--ERROR ER_UNACCEPTABLE_MUTUAL_RECURSION +set @var= +( + with recursive + cte_1 as ( + select 1 + union + select * from cte_2), + cte_2 as ( + select * from cte_1 + union + select a from t1, cte_2 + where t1.a=cte_2.a) + select * from cte_2 + limit 1 +); + +drop table t1; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index d964831a098..7a4530082eb 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4818,6 +4818,9 @@ end_with_restore_list: { List *lex_var_list= &lex->var_list; + if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) + goto error; + if ((check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE) || open_and_lock_tables(thd, all_tables, TRUE, 0))) goto error; From 252e690c859b53c51c7eecf072f33dbaaa01bcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 19 Dec 2017 16:13:35 +0200 Subject: [PATCH 22/23] Fix galera.view test case crash. WSREP_TO_ISOLATION_BEGIN() call must be after view name is back on tables list. --- sql/sql_view.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 629dd865b0e..0f08883639a 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -425,11 +425,12 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, if ((res= create_view_precheck(thd, tables, view, mode))) goto err; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + /* ignore lock specs for CREATE statement */ From 88aff5f471d3d9ae8ecc2f909bcf5bd0ddd6aa7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 18 Dec 2017 13:38:36 +0200 Subject: [PATCH 23/23] Follow-up to MDEV-13407 innodb.drop_table_background failed in buildbot with "Tablespace for table exists" The InnoDB background DROP TABLE queue is something that we should really remove, but are unable to until we remove dict_operation_lock so that DDL and DML operations can be combined in a single transaction. Because the queue is not persistent, it is not crash-safe. We should in some way ensure that the deferred-dropped tables will be dropped after server restart. The existence of two separate transactions complicates the error handling of CREATE TABLE...SELECT. We should really not break locks in DROP TABLE. Our solution to these problems is to rename the table to a temporary name, and to drop such-named tables on InnoDB startup. Also, the queue will use table IDs instead of names from now on. check-testcase.test: Ignore #sql-ib*.ibd files, because tables may enter the background DROP TABLE queue shortly before the test finishes. innodb.drop_table_background: Test CREATE...SELECT and the creation of tables whose file name starts with #sql-ib. innodb.alter_crash: Adjust the recovery, now that the #sql-ib tables will be dropped on InnoDB startup. row_mysql_drop_garbage_tables(): New function, to drop all #sql-ib tables on InnoDB startup. row_drop_table_for_mysql_in_background(): Remove an unnecessary and misplaced call to log_buffer_flush_to_disk(). (The call should have been after the transaction commit. We do not care about flushing the redo log here, because the table would be dropped again at server startup.) Remove the entry from the list after the table no longer exists. If server shutdown has been initiated, empty the list without actually dropping any tables. They will be dropped again on startup. row_drop_table_for_mysql(): Do not call lock_remove_all_on_table(). Instead, if locks exist, defer the DROP TABLE until they do not exist. If the table name does not start with #sql-ib, rename it to that prefix before adding it to the background DROP TABLE queue. --- mysql-test/include/check-testcase.test | 5 +- mysql-test/suite/innodb/r/alter_crash.result | 21 ++- .../innodb/r/drop_table_background.result | 15 ++ mysql-test/suite/innodb/t/alter_crash.test | 69 ++++----- .../suite/innodb/t/drop_table_background.test | 14 ++ storage/innobase/include/row0mysql.h | 4 + storage/innobase/log/log0recv.cc | 2 + storage/innobase/row/row0mysql.cc | 141 ++++++++++++------ 8 files changed, 166 insertions(+), 105 deletions(-) diff --git a/mysql-test/include/check-testcase.test b/mysql-test/include/check-testcase.test index a282201857e..4ca53989d06 100644 --- a/mysql-test/include/check-testcase.test +++ b/mysql-test/include/check-testcase.test @@ -82,7 +82,10 @@ call mtr.check_testcase(); let $datadir=`select @@datadir`; list_files $datadir mysql_upgrade_info; -list_files $datadir/test #sql*; +list_files_write_file $datadir.tempfiles.txt $datadir/test #sql*; +--replace_regex /#sql-ib[0-9a-f]+-[0-9a-f]+\.ibd\n// +cat_file $datadir.tempfiles.txt; +remove_file $datadir.tempfiles.txt; list_files $datadir/mysql #sql*; --enable_query_log diff --git a/mysql-test/suite/innodb/r/alter_crash.result b/mysql-test/suite/innodb/r/alter_crash.result index 8de02cc5fbd..df1645a4ef6 100644 --- a/mysql-test/suite/innodb/r/alter_crash.result +++ b/mysql-test/suite/innodb/r/alter_crash.result @@ -44,10 +44,9 @@ SET DEBUG_DBUG='+d,innodb_alter_commit_crash_after_commit'; ALTER TABLE t1 ADD PRIMARY KEY (f2, f1); ERROR HY000: Lost connection to MySQL server during query # Restart mysqld after the crash and reconnect. -# Manual *.frm recovery begin. -# Manual recovery end -FLUSH TABLES; -# Drop the orphaned original table. +SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = ID; +TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE # Files in datadir after manual recovery. t1.frm t1.ibd @@ -83,11 +82,9 @@ SET DEBUG_DBUG='+d,innodb_alter_commit_crash_before_commit'; ALTER TABLE t2 ADD PRIMARY KEY (f2, f1); ERROR HY000: Lost connection to MySQL server during query # Startup the server after the crash -# Read and remember the temporary table name -# Manual *.frm recovery begin. The dictionary was not updated -# and the files were not renamed. The rebuilt table -# was left behind on purpose, to faciliate data recovery. -# Manual recovery end +SELECT * FROM information_schema.innodb_sys_tables +WHERE name LIKE 'test/#sql-ib%'; +TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE # Drop the orphaned rebuilt table. SHOW TABLES; Tables_in_test @@ -123,10 +120,10 @@ SET DEBUG_DBUG='+d,innodb_alter_commit_crash_after_commit'; ALTER TABLE t1 ADD INDEX (b), CHANGE c d int, ALGORITHM=INPLACE; ERROR HY000: Lost connection to MySQL server during query # Restart mysqld after the crash and reconnect. -# Manual *.frm recovery begin. -# Manual recovery end +SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = ID; +TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE FLUSH TABLES; -# Drop the orphaned original table. # Files in datadir after manual recovery. t1.frm t1.ibd diff --git a/mysql-test/suite/innodb/r/drop_table_background.result b/mysql-test/suite/innodb/r/drop_table_background.result index a6f5672ba7f..e74bcd5e780 100644 --- a/mysql-test/suite/innodb/r/drop_table_background.result +++ b/mysql-test/suite/innodb/r/drop_table_background.result @@ -3,7 +3,22 @@ KEY(c1), KEY(c2), KEY(c2,c1), KEY(c3), KEY(c3,c1), KEY(c3,c2), KEY(c3,c2,c1), KEY(c4), KEY(c4,c1), KEY(c4,c2), KEY(c4,c2,c1), KEY(c4,c3), KEY(c4,c3,c1), KEY(c4,c3,c2), KEY(c4,c3,c2,c1)) ENGINE=InnoDB; +CREATE TABLE `#mysql50##sql-ib-foo`(a SERIAL) ENGINE=InnoDB; +INSERT INTO t (c1) VALUES (1),(2),(1); SET DEBUG_DBUG='+d,row_drop_table_add_to_background'; +CREATE TABLE target (PRIMARY KEY(c1)) ENGINE=InnoDB SELECT * FROM t; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +SELECT * from target; +ERROR 42S02: Table 'test.target' doesn't exist DROP TABLE t; CREATE TABLE t (a INT) ENGINE=InnoDB; DROP TABLE t; +DROP TABLE target; +ERROR 42S02: Unknown table 'test.target' +CREATE TABLE target (a INT) ENGINE=InnoDB; +DROP TABLE target; +SELECT * FROM `#mysql50##sql-ib-foo`; +ERROR 42S02: Table 'test.#mysql50##sql-ib-foo' doesn't exist in engine +DROP TABLE `#mysql50##sql-ib-foo`; +Warnings: +Warning 1932 Table 'test.#mysql50##sql-ib-foo' doesn't exist in engine diff --git a/mysql-test/suite/innodb/t/alter_crash.test b/mysql-test/suite/innodb/t/alter_crash.test index b4fdfd2f2d5..c4ee895d192 100644 --- a/mysql-test/suite/innodb/t/alter_crash.test +++ b/mysql-test/suite/innodb/t/alter_crash.test @@ -75,28 +75,22 @@ ALTER TABLE t1 ADD PRIMARY KEY (f2, f1); --echo # Restart mysqld after the crash and reconnect. --source include/start_mysqld.inc -let $temp_table_name = `SELECT SUBSTR(name, 6) - FROM information_schema.innodb_sys_tables - WHERE table_id = $orig_table_id`; - ---echo # Manual *.frm recovery begin. - ---move_file $MYSQLD_DATADIR/test/t1.frm $MYSQLD_DATADIR/test/$temp_table_name.frm - +let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc; perl; -my @frm_file = glob "$ENV{'datadir'}/test/#sql-*.frm"; -my $t1_frm = "$ENV{'datadir'}/test/t1.frm"; -rename($frm_file[0], $t1_frm); +die unless open OUT, ">$ENV{TABLENAME_INC}"; +chdir "$ENV{'datadir'}/test"; +my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm"; +print OUT 'let $tablename=', $frm_file[0], ';'; +close OUT or die; EOF +source $TABLENAME_INC; +remove_file $TABLENAME_INC; ---echo # Manual recovery end +--replace_result $orig_table_id ID +eval SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = $orig_table_id; -FLUSH TABLES; - ---echo # Drop the orphaned original table. ---disable_query_log -eval DROP TABLE `#mysql50#$temp_table_name`; ---enable_query_log +move_file $datadir/test/$tablename.frm $datadir/test/t1.frm; --echo # Files in datadir after manual recovery. --list_files $MYSQLD_DATADIR/test @@ -134,16 +128,9 @@ ALTER TABLE t2 ADD PRIMARY KEY (f2, f1); --echo # Startup the server after the crash --source include/start_mysqld.inc ---echo # Read and remember the temporary table name -let $temp_table_name = `SELECT SUBSTRING(name,6) - FROM information_schema.innodb_sys_tables - WHERE name LIKE "test/#sql-ib$orig_table_id%"`; +SELECT * FROM information_schema.innodb_sys_tables +WHERE name LIKE 'test/#sql-ib%'; ---echo # Manual *.frm recovery begin. The dictionary was not updated ---echo # and the files were not renamed. The rebuilt table ---echo # was left behind on purpose, to faciliate data recovery. - -let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc; perl; die unless open OUT, ">$ENV{TABLENAME_INC}"; chdir "$ENV{'datadir'}/test"; @@ -154,8 +141,6 @@ EOF source $TABLENAME_INC; remove_file $TABLENAME_INC; ---echo # Manual recovery end - --echo # Drop the orphaned rebuilt table. --disable_query_log eval DROP TABLE `#mysql50#$tablename`; @@ -198,28 +183,24 @@ ALTER TABLE t1 ADD INDEX (b), CHANGE c d int, ALGORITHM=INPLACE; --echo # Restart mysqld after the crash and reconnect. --source include/start_mysqld.inc -let $temp_table_name = `SELECT SUBSTR(name, 6) - FROM information_schema.innodb_sys_tables - WHERE table_id = $orig_table_id`; - ---echo # Manual *.frm recovery begin. ---move_file $MYSQLD_DATADIR/test/t1.frm $MYSQLD_DATADIR/test/$temp_table_name.frm +--replace_result $orig_table_id ID +eval SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = $orig_table_id; perl; -my @frm_file = glob "$ENV{'datadir'}/test/#sql-*.frm"; -my $t1_frm = "$ENV{'datadir'}/test/t1.frm"; -rename($frm_file[0], $t1_frm); +die unless open OUT, ">$ENV{TABLENAME_INC}"; +chdir "$ENV{'datadir'}/test"; +my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm"; +print OUT 'let $tablename=', $frm_file[0], ';'; +close OUT or die; EOF ---echo # Manual recovery end +source $TABLENAME_INC; +remove_file $TABLENAME_INC; +move_file $datadir/test/$tablename.frm $datadir/test/t1.frm; FLUSH TABLES; ---echo # Drop the orphaned original table. ---disable_query_log -eval DROP TABLE `#mysql50#$temp_table_name`; ---enable_query_log - --echo # Files in datadir after manual recovery. --list_files $MYSQLD_DATADIR/test diff --git a/mysql-test/suite/innodb/t/drop_table_background.test b/mysql-test/suite/innodb/t/drop_table_background.test index 0f596dec574..8d82bea9675 100644 --- a/mysql-test/suite/innodb/t/drop_table_background.test +++ b/mysql-test/suite/innodb/t/drop_table_background.test @@ -9,6 +9,9 @@ KEY(c3), KEY(c3,c1), KEY(c3,c2), KEY(c3,c2,c1), KEY(c4), KEY(c4,c1), KEY(c4,c2), KEY(c4,c2,c1), KEY(c4,c3), KEY(c4,c3,c1), KEY(c4,c3,c2), KEY(c4,c3,c2,c1)) ENGINE=InnoDB; +CREATE TABLE `#mysql50##sql-ib-foo`(a SERIAL) ENGINE=InnoDB; +INSERT INTO t (c1) VALUES (1),(2),(1); + let $n= 10; SET DEBUG_DBUG='+d,row_drop_table_add_to_background'; @@ -24,7 +27,18 @@ while ($i) { dec $i; } --enable_query_log +--error ER_DUP_ENTRY +CREATE TABLE target (PRIMARY KEY(c1)) ENGINE=InnoDB SELECT * FROM t; +--error ER_NO_SUCH_TABLE +SELECT * from target; DROP TABLE t; --source include/restart_mysqld.inc CREATE TABLE t (a INT) ENGINE=InnoDB; DROP TABLE t; +--error ER_BAD_TABLE_ERROR +DROP TABLE target; +CREATE TABLE target (a INT) ENGINE=InnoDB; +DROP TABLE target; +--error ER_NO_SUCH_TABLE_IN_ENGINE +SELECT * FROM `#mysql50##sql-ib-foo`; +DROP TABLE `#mysql50##sql-ib-foo`; diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index a7a55d202e8..044e732c22d 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -420,6 +420,10 @@ ulint row_get_background_drop_list_len_low(void); /*======================================*/ +/** Drop garbage tables during recovery. */ +void +row_mysql_drop_garbage_tables(); + /*********************************************************************//** Sets an exclusive lock on a table. @return error code or DB_SUCCESS */ diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 2e967a99121..93943620ecf 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3410,6 +3410,8 @@ recv_recovery_rollback_active(void) /* Drop partially created indexes. */ row_merge_drop_temp_indexes(); + /* Drop garbage tables. */ + row_mysql_drop_garbage_tables(); /* Drop any auxiliary tables that were not dropped when the parent table was dropped. This can happen if the parent table diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index e6ef1caa7ff..6d8b0b968c6 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -64,6 +64,7 @@ Created 9/17/2000 Heikki Tuuri #include "trx0roll.h" #include "trx0undo.h" #include "row0ext.h" +#include "srv0start.h" #include "ut0new.h" #include @@ -2776,12 +2777,6 @@ row_drop_table_for_mysql_in_background( error = row_drop_table_for_mysql(name, trx, FALSE, FALSE); - /* Flush the log to reduce probability that the .frm files and - the InnoDB data dictionary get out-of-sync if the user runs - with innodb_flush_log_at_trx_commit = 0 */ - - log_buffer_flush_to_disk(); - trx_commit_for_mysql(trx); trx_free_for_background(trx); @@ -2819,8 +2814,11 @@ next: return(n_tables + n_tables_dropped); } - table = dict_table_open_on_id(drop->table_id, FALSE, - DICT_TABLE_OP_OPEN_ONLY_IF_CACHED); + /* On fast shutdown, just empty the list without dropping tables. */ + table = srv_shutdown_state == SRV_SHUTDOWN_NONE || !srv_fast_shutdown + ? dict_table_open_on_id(drop->table_id, FALSE, + DICT_TABLE_OP_OPEN_ONLY_IF_CACHED) + : NULL; if (!table) { n_tables_dropped++; @@ -2875,6 +2873,74 @@ row_get_background_drop_list_len_low(void) return(len); } +/** Drop garbage tables during recovery. */ +void +row_mysql_drop_garbage_tables() +{ + mem_heap_t* heap = mem_heap_create(FN_REFLEN); + btr_pcur_t pcur; + mtr_t mtr; + trx_t* trx = trx_allocate_for_background(); + trx->op_info = "dropping garbage tables"; + row_mysql_lock_data_dictionary(trx); + + mtr.start(); + btr_pcur_open_at_index_side( + true, dict_table_get_first_index(dict_sys->sys_tables), + BTR_SEARCH_LEAF, &pcur, true, 0, &mtr); + + for (;;) { + const rec_t* rec; + const byte* field; + ulint len; + const char* table_name; + + btr_pcur_move_to_next_user_rec(&pcur, &mtr); + + if (!btr_pcur_is_on_user_rec(&pcur)) { + break; + } + + rec = btr_pcur_get_rec(&pcur); + if (rec_get_deleted_flag(rec, 0)) { + continue; + } + + field = rec_get_nth_field_old(rec, 0/*NAME*/, &len); + if (len == UNIV_SQL_NULL || len == 0) { + /* Corrupted SYS_TABLES.NAME */ + continue; + } + + table_name = mem_heap_strdupl( + heap, + reinterpret_cast(field), len); + if (strstr(table_name, "/" TEMP_FILE_PREFIX_INNODB)) { + btr_pcur_store_position(&pcur, &mtr); + btr_pcur_commit_specify_mtr(&pcur, &mtr); + + if (dict_load_table(table_name, true, + DICT_ERR_IGNORE_ALL)) { + row_drop_table_for_mysql( + table_name, trx, FALSE, FALSE); + trx_commit_for_mysql(trx); + } + + mtr.start(); + btr_pcur_restore_position(BTR_SEARCH_LEAF, + &pcur, &mtr); + } + + mem_heap_empty(heap); + } + + btr_pcur_close(&pcur); + mtr.commit(); + row_mysql_unlock_data_dictionary(trx); + trx_free_for_background(trx); + mem_heap_free(heap); +} + /*********************************************************************//** If a table is not yet in the drop list, adds the table to the list of tables which the master thread drops in background. We need this on Unix because in @@ -3645,11 +3711,7 @@ row_drop_table_for_mysql( } - DBUG_EXECUTE_IF("row_drop_table_add_to_background", - row_add_table_to_background_drop_list(table->id); - err = DB_SUCCESS; - goto funct_exit; - ); + DBUG_EXECUTE_IF("row_drop_table_add_to_background", goto defer;); /* TODO: could we replace the counter n_foreign_key_checks_running with lock checks on the table? Acquire here an exclusive lock on the @@ -3658,17 +3720,22 @@ row_drop_table_for_mysql( checks take an IS or IX lock on the table. */ if (table->n_foreign_key_checks_running > 0) { - if (row_add_table_to_background_drop_list(table->id)) { - ib::info() << "You are trying to drop table " - << table->name - << " though there is a foreign key check" - " running on it. Adding the table to the" - " background drop queue."; +defer: + if (!strstr(table->name.m_name, "/" TEMP_FILE_PREFIX_INNODB)) { + heap = mem_heap_create(FN_REFLEN); + const char* tmp_name + = dict_mem_create_temporary_tablename( + heap, table->name.m_name, table->id); + ib::info() << "Deferring DROP TABLE " << table->name + << "; renaming to " << tmp_name; + err = row_rename_table_for_mysql( + table->name.m_name, tmp_name, trx, false); + } else { + err = DB_SUCCESS; + } + if (err == DB_SUCCESS) { + row_add_table_to_background_drop_list(table->id); } - - /* We return DB_SUCCESS to MySQL though the drop will - happen lazily later */ - err = DB_SUCCESS; goto funct_exit; } @@ -3689,26 +3756,9 @@ row_drop_table_for_mysql( /* Wait on background threads to stop using table */ fil_wait_crypt_bg_threads(table); - if (table->get_ref_count() == 0) { - lock_remove_all_on_table(table, TRUE); - ut_a(table->n_rec_locks == 0); - } else if (table->get_ref_count() > 0 || table->n_rec_locks > 0) { - if (row_add_table_to_background_drop_list(table->id)) { - ib::info() << "MySQL is trying to drop table " - << table->name - << " though there are still open handles to" - " it. Adding the table to the background drop" - " queue."; - - /* We return DB_SUCCESS to MySQL though the drop will - happen lazily later */ - err = DB_SUCCESS; - } else { - /* The table is already in the background drop list */ - err = DB_ERROR; - } - - goto funct_exit; + if (table->get_ref_count() > 0 || table->n_rec_locks > 0 + || lock_table_has_locks(table)) { + goto defer; } /* The "to_be_dropped" marks table that is to be dropped, but @@ -3718,11 +3768,6 @@ row_drop_table_for_mysql( and it is free to be dropped */ table->to_be_dropped = false; - /* If we get this far then the table to be dropped must not have - any table or record locks on it. */ - - ut_a(!lock_table_has_locks(table)); - switch (trx_get_dict_operation(trx)) { case TRX_DICT_OP_NONE: trx_set_dict_operation(trx, TRX_DICT_OP_TABLE);