From c1698e8dc50f0c342c1a6886426ba1d43396cd1e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 29 May 2018 16:47:47 +0100 Subject: [PATCH 1/8] MDEV-16303 - do not install wsrep.ini on Windows --- support-files/CMakeLists.txt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index 3b6e86030ab..dff4610d490 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -41,13 +41,20 @@ ELSE() SET(inst_location ${INSTALL_SUPPORTFILESDIR}) ENDIF() -FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small wsrep) - CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${inifile}.cnf.sh +FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${inifile}.cnf.sh ${CMAKE_CURRENT_BINARY_DIR}/${inifile}.${ini_file_extension} @ONLY) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${inifile}.${ini_file_extension} DESTINATION ${inst_location} COMPONENT IniFiles) ENDFOREACH() +IF(WITH_WSREP) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/wsrep.cnf.sh + ${CMAKE_CURRENT_BINARY_DIR}/wsrep.${ini_file_extension} @ONLY) + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/wsrep.${ini_file_extension} + DESTINATION ${inst_location} COMPONENT IniFiles) +ENDIF() + IF(UNIX) SET(prefix ${CMAKE_INSTALL_PREFIX}) FOREACH(script mysqld_multi.server mysql-log-rotate binary-configure wsrep_notify) From 41cbe92bf116cf35a935496fa8b6c8a9f80a5edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 08:42:24 +0300 Subject: [PATCH 2/8] Remove dead code that was added in MDEV-5834 ha_innobase::set_partition_owner_stats(): Remove (unused function). ha_innobase::ha_partition_stats: Remove (the variable is never read). Remove unused ut_timer functions. --- storage/innobase/btr/btr0defragment.cc | 51 --------------------- storage/innobase/handler/ha_innodb.cc | 10 +---- storage/innobase/handler/ha_innodb.h | 5 +-- storage/innobase/include/ut0timer.h | 39 +--------------- storage/innobase/include/ut0timer.ic | 61 +------------------------- storage/xtradb/btr/btr0defragment.cc | 50 --------------------- storage/xtradb/handler/ha_innodb.cc | 10 +---- storage/xtradb/handler/ha_innodb.h | 5 +-- storage/xtradb/include/ut0timer.h | 39 +--------------- storage/xtradb/include/ut0timer.ic | 61 +------------------------- 10 files changed, 12 insertions(+), 319 deletions(-) diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index ca4c90eef41..9f17815cd43 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -39,57 +39,6 @@ Modified 30/07/2014 Jan Lindström jan.lindstrom@mariadb.com #include -/**************************************************//** -Custom nullptr implementation for under g++ 4.6 -*******************************************************/ -// #pragma once -/* -namespace std -{ - // based on SC22/WG21/N2431 = J16/07-0301 - struct nullptr_t - { - template operator any * () const - { - return 0; - } - template operator T any:: * () const - { - return 0; - } - -#ifdef _MSC_VER - struct pad {}; - pad __[sizeof(void*)/sizeof(pad)]; -#else - char __[sizeof(void*)]; -#endif -private: - // nullptr_t();// {} - // nullptr_t(const nullptr_t&); - // void operator = (const nullptr_t&); - void operator &() const; - template void operator +(any) const - { - // I Love MSVC 2005! - } - template void operator -(any) const - { - // I Love MSVC 2005! - } - }; -static const nullptr_t __nullptr = {}; -} - -#ifndef nullptr -#define nullptr std::__nullptr -#endif -*/ - -/**************************************************//** -End of Custom nullptr implementation for under g++ 4.6 -*******************************************************/ - /* When there's no work, either because defragment is disabled, or because no query is submitted, thread checks state every BTR_DEFRAGMENT_SLEEP_IN_USECS.*/ #define BTR_DEFRAGMENT_SLEEP_IN_USECS 1000000 diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 961166fed9a..f2c241d0869 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2849,8 +2849,7 @@ ha_innobase::ha_innobase( (srv_force_primary_key ? HA_REQUIRE_PRIMARY_KEY : 0 ) | HA_CAN_FULLTEXT_EXT | HA_CAN_EXPORT), start_of_scan(0), - num_write_row(0), - ha_partition_stats(NULL) + num_write_row(0) {} /*********************************************************************//** @@ -16749,13 +16748,6 @@ innodb_max_dirty_pages_pct_lwm_update( srv_max_dirty_pages_pct_lwm = in_val; } -UNIV_INTERN -void -ha_innobase::set_partition_owner_stats(ha_statistics *stats) -{ - ha_partition_stats= stats; -} - /************************************************************//** Validate the file format name and return its corresponding id. @return valid file format id */ diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 300f49c93f9..9c838910434 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -100,8 +100,6 @@ class ha_innobase: public handler or undefined */ uint num_write_row; /*!< number of write_row() calls */ - ha_statistics* ha_partition_stats; /*!< stats of the partition owner - handler (if there is one) */ uint store_key_val_for_row(uint keynr, char* buff, uint buff_len, const uchar* record); inline void update_thd(THD* thd); @@ -316,7 +314,6 @@ class ha_innobase: public handler Alter_inplace_info* ha_alter_info, bool commit); /** @} */ - void set_partition_owner_stats(ha_statistics *stats); bool check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes); private: diff --git a/storage/innobase/include/ut0timer.h b/storage/innobase/include/ut0timer.h index f361ae79bf5..3015771b434 100644 --- a/storage/innobase/include/ut0timer.h +++ b/storage/innobase/include/ut0timer.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Facebook, Inc. All Rights Reserved. -Copyright (c) 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,7 +19,7 @@ this program; if not, write to the Free Software Foundation, Inc., /********************************************************************//** @file include/ut0timer.h -Timer rountines +Timer routines Created 30/07/2014 Jan Lindström jan.lindstrom@skysql.com modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11d7269bdfea06f96d6b6 @@ -28,8 +28,6 @@ modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11 #define ut0timer_h #include "univ.i" -#include "data0type.h" -#include /* Current timer stats */ extern struct my_timer_unit_info ut_timer; @@ -47,39 +45,6 @@ Initializes my_timer struct to contain the info for selected timer.*/ UNIV_INTERN void ut_init_timer(void); -/**************************************************************//** -Return time passed since time then, automatically adjusted -for the estimated timer overhead. -@return time passed since "then" */ -UNIV_INLINE -ulonglong -ut_timer_since( -/*===========*/ - ulonglong then); /*!< in: time where to calculate */ -/**************************************************************//** -Get time passed since "then", and update then to now -@return time passed sinche "then" */ -UNIV_INLINE -ulonglong -ut_timer_since_and_update( -/*======================*/ - ulonglong *then); /*!< in: time where to calculate */ -/**************************************************************//** -Convert native timer units in a ulonglong into seconds in a double -@return time in a seconds */ -UNIV_INLINE -double -ut_timer_to_seconds( -/*=================*/ - ulonglong when); /*!< in: time where to calculate */ -/**************************************************************//** -Convert native timer units in a ulonglong into milliseconds in a double -@return time in milliseconds */ -UNIV_INLINE -double -ut_timer_to_milliseconds( -/*=====================*/ - ulonglong when); /*!< in: time where to calculate */ /**************************************************************//** Convert native timer units in a ulonglong into microseconds in a double @return time in microseconds */ diff --git a/storage/innobase/include/ut0timer.ic b/storage/innobase/include/ut0timer.ic index 46dcd0cb718..34886481efa 100644 --- a/storage/innobase/include/ut0timer.ic +++ b/storage/innobase/include/ut0timer.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Facebook, Inc. All Rights Reserved. -Copyright (c) 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,69 +19,12 @@ this program; if not, write to the Free Software Foundation, Inc., /********************************************************************//** @file include/ut0timer.ic -Timer rountines +Timer routines Created 30/07/2014 Jan Lindström jan.lindstrom@skysql.com modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11d7269bdfea06f96d6b6 *************************************************************************/ -/**************************************************************//** -Return time passed since time then, automatically adjusted -for the estimated timer overhead. -@return time passed since "then" */ -UNIV_INLINE -ulonglong -ut_timer_since( -/*===========*/ - ulonglong then) /*!< in: time where to calculate */ -{ - return (ut_timer_now() - then) - ut_timer.overhead; -} - -/**************************************************************//** -Get time passed since "then", and update then to now -@return time passed sinche "then" */ -UNIV_INLINE -ulonglong -ut_timer_since_and_update( -/*======================*/ - ulonglong *then) /*!< in: time where to calculate */ -{ - ulonglong now = ut_timer_now(); - ulonglong ret = (now - (*then)) - ut_timer.overhead; - *then = now; - return ret; -} - -/**************************************************************//** -Convert native timer units in a ulonglong into seconds in a double -@return time in a seconds */ -UNIV_INLINE -double -ut_timer_to_seconds( -/*=================*/ - ulonglong when) /*!< in: time where to calculate */ -{ - double ret = (double)(when); - ret /= (double)(ut_timer.frequency); - return ret; -} - -/**************************************************************//** -Convert native timer units in a ulonglong into milliseconds in a double -@return time in milliseconds */ -UNIV_INLINE -double -ut_timer_to_milliseconds( -/*=====================*/ - ulonglong when) /*!< in: time where to calculate */ -{ - double ret = (double)(when); - ret *= 1000.0; - ret /= (double)(ut_timer.frequency); - return ret; -} - /**************************************************************//** Convert native timer units in a ulonglong into microseconds in a double @return time in microseconds */ diff --git a/storage/xtradb/btr/btr0defragment.cc b/storage/xtradb/btr/btr0defragment.cc index 44acd9118d7..3124db650c6 100644 --- a/storage/xtradb/btr/btr0defragment.cc +++ b/storage/xtradb/btr/btr0defragment.cc @@ -40,56 +40,6 @@ Modified 30/07/2014 Jan Lindström jan.lindstrom@mariadb.com #include -/**************************************************//** -Custom nullptr implementation for under g++ 4.6 -*******************************************************/ -/* -// #pragma once -namespace std -{ - // based on SC22/WG21/N2431 = J16/07-0301 - struct nullptr_t - { - template operator any * () const - { - return 0; - } - template operator T any:: * () const - { - return 0; - } - -#ifdef _MSC_VER - struct pad {}; - pad __[sizeof(void*)/sizeof(pad)]; -#else - char __[sizeof(void*)]; -#endif -private: - // nullptr_t();// {} - // nullptr_t(const nullptr_t&); - // void operator = (const nullptr_t&); - void operator &() const; - template void operator +(any) const - { - // I Love MSVC 2005! - } - template void operator -(any) const - { - // I Love MSVC 2005! - } - }; -static const nullptr_t __nullptr = {}; -} - -#ifndef nullptr -#define nullptr std::__nullptr -#endif -*/ -/**************************************************//** -End of Custom nullptr implementation for under g++ 4.6 -*******************************************************/ - /* When there's no work, either because defragment is disabled, or because no query is submitted, thread checks state every BTR_DEFRAGMENT_SLEEP_IN_USECS.*/ #define BTR_DEFRAGMENT_SLEEP_IN_USECS 1000000 diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 238627ce003..a673dd78c70 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3259,8 +3259,7 @@ ha_innobase::ha_innobase( (srv_force_primary_key ? HA_REQUIRE_PRIMARY_KEY : 0 ) | HA_CAN_FULLTEXT_EXT | HA_CAN_EXPORT), start_of_scan(0), - num_write_row(0), - ha_partition_stats(NULL) + num_write_row(0) {} /*********************************************************************//** @@ -17527,13 +17526,6 @@ innodb_max_dirty_pages_pct_lwm_update( srv_max_dirty_pages_pct_lwm = in_val; } -UNIV_INTERN -void -ha_innobase::set_partition_owner_stats(ha_statistics *stats) -{ - ha_partition_stats= stats; -} - /************************************************************//** Validate the file format name and return its corresponding id. @return valid file format id */ diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index c5b0e723702..3dbbc53a0e3 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -100,8 +100,6 @@ class ha_innobase: public handler or undefined */ uint num_write_row; /*!< number of write_row() calls */ - ha_statistics* ha_partition_stats; /*!< stats of the partition owner - handler (if there is one) */ uint store_key_val_for_row(uint keynr, char* buff, uint buff_len, const uchar* record); inline void update_thd(THD* thd); @@ -318,7 +316,6 @@ class ha_innobase: public handler Alter_inplace_info* ha_alter_info, bool commit); /** @} */ - void set_partition_owner_stats(ha_statistics *stats); bool check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes); diff --git a/storage/xtradb/include/ut0timer.h b/storage/xtradb/include/ut0timer.h index f361ae79bf5..3015771b434 100644 --- a/storage/xtradb/include/ut0timer.h +++ b/storage/xtradb/include/ut0timer.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Facebook, Inc. All Rights Reserved. -Copyright (c) 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,7 +19,7 @@ this program; if not, write to the Free Software Foundation, Inc., /********************************************************************//** @file include/ut0timer.h -Timer rountines +Timer routines Created 30/07/2014 Jan Lindström jan.lindstrom@skysql.com modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11d7269bdfea06f96d6b6 @@ -28,8 +28,6 @@ modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11 #define ut0timer_h #include "univ.i" -#include "data0type.h" -#include /* Current timer stats */ extern struct my_timer_unit_info ut_timer; @@ -47,39 +45,6 @@ Initializes my_timer struct to contain the info for selected timer.*/ UNIV_INTERN void ut_init_timer(void); -/**************************************************************//** -Return time passed since time then, automatically adjusted -for the estimated timer overhead. -@return time passed since "then" */ -UNIV_INLINE -ulonglong -ut_timer_since( -/*===========*/ - ulonglong then); /*!< in: time where to calculate */ -/**************************************************************//** -Get time passed since "then", and update then to now -@return time passed sinche "then" */ -UNIV_INLINE -ulonglong -ut_timer_since_and_update( -/*======================*/ - ulonglong *then); /*!< in: time where to calculate */ -/**************************************************************//** -Convert native timer units in a ulonglong into seconds in a double -@return time in a seconds */ -UNIV_INLINE -double -ut_timer_to_seconds( -/*=================*/ - ulonglong when); /*!< in: time where to calculate */ -/**************************************************************//** -Convert native timer units in a ulonglong into milliseconds in a double -@return time in milliseconds */ -UNIV_INLINE -double -ut_timer_to_milliseconds( -/*=====================*/ - ulonglong when); /*!< in: time where to calculate */ /**************************************************************//** Convert native timer units in a ulonglong into microseconds in a double @return time in microseconds */ diff --git a/storage/xtradb/include/ut0timer.ic b/storage/xtradb/include/ut0timer.ic index 815726e9d0a..02af23cdc6d 100644 --- a/storage/xtradb/include/ut0timer.ic +++ b/storage/xtradb/include/ut0timer.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Facebook, Inc. All Rights Reserved. -Copyright (c) 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,69 +19,12 @@ this program; if not, write to the Free Software Foundation, Inc., /********************************************************************//** @file include/ut0timer.ic -Timer rountines +Timer routines Created 30/07/2014 Jan Lindström jan.lindstrom@skysql.com modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11d7269bdfea06f96d6b6 *************************************************************************/ -/**************************************************************//** -Return time passed since time then, automatically adjusted -for the estimated timer overhead. -@return time passed since "then" */ -UNIV_INLINE -ulonglong -ut_timer_since( -/*===========*/ - ulonglong then) /*!< in: time where to calculate */ -{ - return (ut_timer_now() - then) - ut_timer.overhead; -} - -/**************************************************************//** -Get time passed since "then", and update then to now -@return time passed sinche "then" */ -UNIV_INLINE -ulonglong -ut_timer_since_and_update( -/*======================*/ - ulonglong *then) /*!< in: time where to calculate */ -{ - ulonglong now = ut_timer_now(); - ulonglong ret = (now - (*then)) - ut_timer.overhead; - *then = now; - return ret; -} - -/**************************************************************//** -Convert native timer units in a ulonglong into seconds in a double -@return time in a seconds */ -UNIV_INLINE -double -ut_timer_to_seconds( -/*=================*/ - ulonglong when) /*!< in: time where to calculate */ -{ - double ret = (double)(when); - ret /= (double)(ut_timer.frequency); - return ret; -} - -/**************************************************************//** -Convert native timer units in a ulonglong into milliseconds in a double -@return time in milliseconds */ -UNIV_INLINE -double -ut_timer_to_milliseconds( -/*=====================*/ - ulonglong when) /*!< in: time where to calculate */ -{ - double ret = (double)(when); - ret *= 1000.0; - ret /= (double)(ut_timer.frequency); - return ret; -} - /**************************************************************//** Convert native timer units in a ulonglong into microseconds in a double @return time in microseconds */ From f6376bfd1c25997283cbaf4eb835cc1e09e49def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 09:25:34 +0300 Subject: [PATCH 3/8] Add FLUSH TABLES to avoid corruption of MyISAM system tables --- mysql-test/suite/innodb/r/innodb-blob.result | 1 + mysql-test/suite/innodb/t/innodb-blob.test | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/innodb/r/innodb-blob.result b/mysql-test/suite/innodb/r/innodb-blob.result index eca77972400..5d2101d8eeb 100644 --- a/mysql-test/suite/innodb/r/innodb-blob.result +++ b/mysql-test/suite/innodb/r/innodb-blob.result @@ -1,3 +1,4 @@ +FLUSH TABLES; CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=InnoDB; CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t3 (a INT PRIMARY KEY, b TEXT, c TEXT) ENGINE=InnoDB; diff --git a/mysql-test/suite/innodb/t/innodb-blob.test b/mysql-test/suite/innodb/t/innodb-blob.test index 775d52c9ce8..17fd194dbeb 100644 --- a/mysql-test/suite/innodb/t/innodb-blob.test +++ b/mysql-test/suite/innodb/t/innodb-blob.test @@ -5,12 +5,14 @@ --source include/have_innodb.inc --source include/have_innodb_16k.inc -# DEBUG_SYNC must be compiled in. +--source include/have_debug.inc --source include/have_debug_sync.inc # Embedded server does not support restarting --source include/not_embedded.inc +FLUSH TABLES; + CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=InnoDB; CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t3 (a INT PRIMARY KEY, b TEXT, c TEXT) ENGINE=InnoDB; From 3b7da8a44c8a0ff4b40b37e4db01f7e397aefab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 10:36:25 +0300 Subject: [PATCH 4/8] MDEV-15824 innodb_defragment=ON trumps innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE ha_innobase::optimize(): If both innodb_defragment and innodb_optimize_fulltext_only are at their default settings (OFF), fall back to ALTER TABLE. Else process one or both options. --- .../innodb/r/innodb_defragment_small.result | 21 +++++++++++++---- .../innodb/t/innodb_defragment_small.test | 23 +++++++++++++------ storage/innobase/handler/ha_innodb.cc | 14 +++++------ storage/xtradb/handler/ha_innodb.cc | 14 +++++------ 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_defragment_small.result b/mysql-test/suite/innodb/r/innodb_defragment_small.result index 3f594f79e70..07e58f442d9 100644 --- a/mysql-test/suite/innodb/r/innodb_defragment_small.result +++ b/mysql-test/suite/innodb/r/innodb_defragment_small.result @@ -1,6 +1,13 @@ SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment; +SET @innodb_optimize_fulltext_orig=@@GLOBAL.innodb_optimize_fulltext_only; SET GLOBAL innodb_defragment = 1; -CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB; +SET GLOBAL innodb_optimize_fulltext_only = 0; +# +# MDEV-12198 innodb_defragment=1 crashes server on +# OPTIMIZE TABLE when FULLTEXT index exists +# +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), +KEY(a, b), FULLTEXT KEY(b)) ENGINE=INNODB; OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize status OK @@ -11,14 +18,18 @@ INSERT INTO t1 VALUES (400000, REPEAT('A', 256)); OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize status OK -DROP TABLE t1; # -# MDEV-12198 innodb_defragment=1 crashes server on -# OPTIMIZE TABLE when FULLTEXT index exists +# MDEV-15824 innodb_defragment=ON trumps +# innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE # -CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB; +SET GLOBAL innodb_optimize_fulltext_only = 1; +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +SET GLOBAL innodb_defragment = 0; OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize status OK DROP TABLE t1; SET GLOBAL innodb_defragment = @innodb_defragment_orig; +SET GLOBAL innodb_optimize_fulltext_only = @innodb_optimize_fulltext_orig; diff --git a/mysql-test/suite/innodb/t/innodb_defragment_small.test b/mysql-test/suite/innodb/t/innodb_defragment_small.test index 454333d6986..c52a58fc5a7 100644 --- a/mysql-test/suite/innodb/t/innodb_defragment_small.test +++ b/mysql-test/suite/innodb/t/innodb_defragment_small.test @@ -1,10 +1,17 @@ --source include/have_innodb.inc SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment; +SET @innodb_optimize_fulltext_orig=@@GLOBAL.innodb_optimize_fulltext_only; SET GLOBAL innodb_defragment = 1; +SET GLOBAL innodb_optimize_fulltext_only = 0; -# Small tests copied from innodb.innodb_defragment -CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB; +--echo # +--echo # MDEV-12198 innodb_defragment=1 crashes server on +--echo # OPTIMIZE TABLE when FULLTEXT index exists +--echo # + +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), + KEY(a, b), FULLTEXT KEY(b)) ENGINE=INNODB; OPTIMIZE TABLE t1; INSERT INTO t1 VALUES (100000, REPEAT('A', 256)); @@ -13,16 +20,18 @@ INSERT INTO t1 VALUES (300000, REPEAT('A', 256)); INSERT INTO t1 VALUES (400000, REPEAT('A', 256)); OPTIMIZE TABLE t1; -DROP TABLE t1; --echo # ---echo # MDEV-12198 innodb_defragment=1 crashes server on ---echo # OPTIMIZE TABLE when FULLTEXT index exists +--echo # MDEV-15824 innodb_defragment=ON trumps +--echo # innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE --echo # -CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB; - +SET GLOBAL innodb_optimize_fulltext_only = 1; OPTIMIZE TABLE t1; +SET GLOBAL innodb_defragment = 0; +OPTIMIZE TABLE t1; + DROP TABLE t1; SET GLOBAL innodb_defragment = @innodb_defragment_orig; +SET GLOBAL innodb_optimize_fulltext_only = @innodb_optimize_fulltext_orig; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index f2c241d0869..c525c2f0efa 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13904,6 +13904,7 @@ ha_innobase::optimize( This works OK otherwise, but MySQL locks the entire table during calls to OPTIMIZE, which is undesirable. */ + bool try_alter = true; if (srv_defragment) { int err; @@ -13911,7 +13912,7 @@ ha_innobase::optimize( err = defragment_table(prebuilt->table->name, NULL, false); if (err == 0) { - return (HA_ADMIN_OK); + try_alter = false; } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, err, @@ -13919,9 +13920,7 @@ ha_innobase::optimize( prebuilt->table->name, err); if(err == ER_SP_ALREADY_EXISTS) { - return (HA_ADMIN_OK); - } else { - return (HA_ADMIN_TRY_ALTER); + try_alter = false; } } } @@ -13932,11 +13931,10 @@ ha_innobase::optimize( fts_sync_table(prebuilt->table, false, true, false); fts_optimize_table(prebuilt->table); } - return(HA_ADMIN_OK); - } else { - - return(HA_ADMIN_TRY_ALTER); + try_alter = false; } + + return try_alter ? HA_ADMIN_TRY_ALTER : HA_ADMIN_OK; } /*******************************************************************//** diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index a673dd78c70..d642f9c0a06 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -14522,6 +14522,7 @@ ha_innobase::optimize( This works OK otherwise, but MySQL locks the entire table during calls to OPTIMIZE, which is undesirable. */ + bool try_alter = true; if (srv_defragment) { int err; @@ -14529,7 +14530,7 @@ ha_innobase::optimize( err = defragment_table(prebuilt->table->name, NULL, false); if (err == 0) { - return (HA_ADMIN_OK); + try_alter = false; } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, err, @@ -14537,9 +14538,7 @@ ha_innobase::optimize( prebuilt->table->name, err); if(err == ER_SP_ALREADY_EXISTS) { - return (HA_ADMIN_OK); - } else { - return (HA_ADMIN_TRY_ALTER); + try_alter = false; } } } @@ -14550,11 +14549,10 @@ ha_innobase::optimize( fts_sync_table(prebuilt->table, false, true, false); fts_optimize_table(prebuilt->table); } - return(HA_ADMIN_OK); - } else { - - return(HA_ADMIN_TRY_ALTER); + try_alter = false; } + + return try_alter ? HA_ADMIN_TRY_ALTER : HA_ADMIN_OK; } /*******************************************************************//** From 738c5c84245f27c839a0713f52452a45f141b3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 13:06:14 +0000 Subject: [PATCH 5/8] MDEV-12642: Build deb source packages on buildbot, just not on Travis-CI --- debian/autobake-deb.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index c26fcaa4e85..d4de8045788 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -107,12 +107,20 @@ dch -b -D ${CODENAME} -v "${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automat echo "Creating package version ${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME} ... " +# On Travis CI, use -b to build binary only packages as there is no need to +# waste time on generating the source package. +if [[ $TRAVIS ]] +then + BUILDPACKAGE_FLAGS="-b" +fi + # Build the package # Pass -I so that .git and other unnecessary temporary and source control files # will be ignored by dpkg-source when creating the tar.gz source package. -# Use -b to build binary only packages as there is no need to waste time on -# generating the source package. -fakeroot dpkg-buildpackage -us -uc -I -b +fakeroot dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS + +# If the step above fails due to missing dependencies, you can manually run +# sudo mk-build-deps debian/control -r -i # Don't log package contents on Travis-CI to save time and log size if [[ ! $TRAVIS ]] From 55abcfa7b70968246a1a26a8839013ebb8f5c506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 18:16:12 +0300 Subject: [PATCH 6/8] MDEV-16124 fil_rename_tablespace() times out and crashes server during table-rebuilding ALTER TABLE InnoDB insisted on closing the file handle before renaming a file. Renaming a file should never be a problem on POSIX systems. Also on Windows it should work if the file was opened in FILE_SHARE_DELETE mode. fil_space_t::stop_ios: Remove. We no longer need to stop file access during rename operations. fil_mutex_enter_and_prepare_for_io(): Remove the wait for stop_ios. fil_rename_tablespace(): Remove the retry logic; do not close the file handle. Remove the unused fault injection that was added along with the DATA DIRECTORY functionality (MySQL WL#5980). os_file_create_simple_func(), os_file_create_func(), os_file_create_simple_no_error_handling_func(): Include FILE_SHARE_DELETE in the share_mode. (We will still prevent multiple InnoDB instances from using the same files by not setting FILE_SHARE_WRITE.) --- storage/innobase/fil/fil0fil.cc | 110 ----------------------------- storage/innobase/include/fil0fil.h | 4 -- storage/innobase/os/os0file.cc | 9 +-- storage/xtradb/fil/fil0fil.cc | 110 ----------------------------- storage/xtradb/include/fil0fil.h | 4 -- storage/xtradb/os/os0file.cc | 9 +-- 6 files changed, 10 insertions(+), 236 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7167b99a750..f852a64e2e9 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -845,7 +845,6 @@ fil_mutex_enter_and_prepare_for_io( ibool success; ibool print_info = FALSE; ulint count = 0; - ulint count2 = 0; retry: mutex_enter(&fil_system->mutex); @@ -862,46 +861,6 @@ retry: space = fil_space_get_by_id(space_id); - if (space != NULL && space->stop_ios) { - /* We are going to do a rename file and want to stop new i/o's - for a while */ - - if (count2 > 20000) { - fputs("InnoDB: Warning: tablespace ", stderr); - ut_print_filename(stderr, space->name); - fprintf(stderr, - " has i/o ops stopped for a long time %lu\n", - (ulong) count2); - } - - mutex_exit(&fil_system->mutex); - -#ifndef UNIV_HOTBACKUP - - /* Wake the i/o-handler threads to make sure pending - i/o's are performed */ - os_aio_simulated_wake_handler_threads(); - - /* The sleep here is just to give IO helper threads a - bit of time to do some work. It is not required that - all IO related to the tablespace being renamed must - be flushed here as we do fil_flush() in - fil_rename_tablespace() as well. */ - os_thread_sleep(20000); - -#endif /* UNIV_HOTBACKUP */ - - /* Flush tablespaces so that we can close modified - files in the LRU list */ - fil_flush_file_spaces(FIL_TABLESPACE); - - os_thread_sleep(20000); - - count2++; - - goto retry; - } - if (fil_system->n_open < fil_system->max_n_open) { return; @@ -2898,7 +2857,6 @@ fil_rename_tablespace( ibool success; fil_space_t* space; fil_node_t* node; - ulint count = 0; char* new_path; char* old_name; char* old_path; @@ -2906,25 +2864,10 @@ fil_rename_tablespace( ut_a(id != 0); -retry: - count++; - - if (!(count % 1000)) { - ut_print_timestamp(stderr); - fputs(" InnoDB: Warning: problems renaming ", stderr); - ut_print_filename(stderr, - old_name_in ? old_name_in : not_given); - fputs(" to ", stderr); - ut_print_filename(stderr, new_name); - fprintf(stderr, ", %lu iterations\n", (ulong) count); - } - mutex_enter(&fil_system->mutex); space = fil_space_get_by_id(id); - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_1", space = NULL; ); - if (space == NULL) { ib_logf(IB_LOG_LEVEL_ERROR, "Cannot find space id %lu in the tablespace " @@ -2936,54 +2879,11 @@ retry: return(FALSE); } - if (count > 25000) { - space->stop_ios = FALSE; - mutex_exit(&fil_system->mutex); - - return(FALSE); - } - - /* We temporarily close the .ibd file because we do not trust that - operating systems can rename an open file. For the closing we have to - wait until there are no pending i/o's or flushes on the file. */ - - space->stop_ios = TRUE; - /* The following code must change when InnoDB supports multiple datafiles per tablespace. */ ut_a(UT_LIST_GET_LEN(space->chain) == 1); node = UT_LIST_GET_FIRST(space->chain); - if (node->n_pending > 0 - || node->n_pending_flushes > 0 - || node->being_extended) { - /* There are pending i/o's or flushes or the file is - currently being extended, sleep for a while and - retry */ - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(20000); - - goto retry; - - } else if (node->modification_counter > node->flush_counter) { - /* Flush the space */ - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(20000); - - fil_flush(id); - - goto retry; - - } else if (node->open) { - /* Close the file */ - - fil_node_close_file(node, fil_system); - } - /* Check that the old name in the space is right */ if (old_name_in) { @@ -3002,17 +2902,9 @@ retry: space, node, new_name, new_path); if (success) { - - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", - goto skip_second_rename; ); - success = os_file_rename( innodb_file_data_key, old_path, new_path); - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", -skip_second_rename: - success = FALSE; ); - if (!success) { /* We have to revert the changes we made to the tablespace memory cache */ @@ -3022,8 +2914,6 @@ skip_second_rename: } } - space->stop_ios = FALSE; - mutex_exit(&fil_system->mutex); #ifndef UNIV_HOTBACKUP diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9374bb88bb7..6e772e31772 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -256,10 +256,6 @@ struct fil_space_t { the space corresponds to a table in the InnoDB data dictionary; so we can print a warning of orphaned tablespaces */ - ibool stop_ios;/*!< TRUE if we want to rename the - .ibd file of tablespace and want to - stop temporarily posting of new i/o - requests on the file */ ibool stop_new_ops; /*!< we set this TRUE when we start deleting a single-table tablespace. diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 55d2c15f906..8f3f3716fc2 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -1186,7 +1186,8 @@ os_file_create_simple_func( /* Use default security attributes and no template file. */ file = CreateFile( - (LPCTSTR) name, access, FILE_SHARE_READ, NULL, + (LPCTSTR) name, access, + FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, create_flag, attributes, NULL); if (file == INVALID_HANDLE_VALUE) { @@ -1314,7 +1315,7 @@ os_file_create_simple_no_error_handling_func( DWORD access; DWORD create_flag; DWORD attributes = 0; - DWORD share_mode = FILE_SHARE_READ; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; ut_a(name); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); @@ -1554,7 +1555,7 @@ os_file_create_func( #ifdef __WIN__ DWORD create_flag; - DWORD share_mode = FILE_SHARE_READ; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? TRUE : FALSE; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 3807e6b3395..49c5f4b090b 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -862,7 +862,6 @@ fil_mutex_enter_and_prepare_for_io( ibool success; ibool print_info = FALSE; ulint count = 0; - ulint count2 = 0; retry: mutex_enter(&fil_system->mutex); @@ -879,46 +878,6 @@ retry: space = fil_space_get_by_id(space_id); - if (space != NULL && space->stop_ios) { - /* We are going to do a rename file and want to stop new i/o's - for a while */ - - if (count2 > 20000) { - fputs("InnoDB: Warning: tablespace ", stderr); - ut_print_filename(stderr, space->name); - fprintf(stderr, - " has i/o ops stopped for a long time %lu\n", - (ulong) count2); - } - - mutex_exit(&fil_system->mutex); - -#ifndef UNIV_HOTBACKUP - - /* Wake the i/o-handler threads to make sure pending - i/o's are performed */ - os_aio_simulated_wake_handler_threads(); - - /* The sleep here is just to give IO helper threads a - bit of time to do some work. It is not required that - all IO related to the tablespace being renamed must - be flushed here as we do fil_flush() in - fil_rename_tablespace() as well. */ - os_thread_sleep(20000); - -#endif /* UNIV_HOTBACKUP */ - - /* Flush tablespaces so that we can close modified - files in the LRU list */ - fil_flush_file_spaces(FIL_TABLESPACE); - - os_thread_sleep(20000); - - count2++; - - goto retry; - } - if (fil_system->n_open < fil_system->max_n_open) { return; @@ -2950,7 +2909,6 @@ fil_rename_tablespace( ibool success; fil_space_t* space; fil_node_t* node; - ulint count = 0; char* new_path; char* old_name; char* old_path; @@ -2958,25 +2916,10 @@ fil_rename_tablespace( ut_a(id != 0); -retry: - count++; - - if (!(count % 1000)) { - ut_print_timestamp(stderr); - fputs(" InnoDB: Warning: problems renaming ", stderr); - ut_print_filename(stderr, - old_name_in ? old_name_in : not_given); - fputs(" to ", stderr); - ut_print_filename(stderr, new_name); - fprintf(stderr, ", %lu iterations\n", (ulong) count); - } - mutex_enter(&fil_system->mutex); space = fil_space_get_by_id(id); - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_1", space = NULL; ); - if (space == NULL) { ib_logf(IB_LOG_LEVEL_ERROR, "Cannot find space id %lu in the tablespace " @@ -2988,54 +2931,11 @@ retry: return(FALSE); } - if (count > 25000) { - space->stop_ios = FALSE; - mutex_exit(&fil_system->mutex); - - return(FALSE); - } - - /* We temporarily close the .ibd file because we do not trust that - operating systems can rename an open file. For the closing we have to - wait until there are no pending i/o's or flushes on the file. */ - - space->stop_ios = TRUE; - /* The following code must change when InnoDB supports multiple datafiles per tablespace. */ ut_a(UT_LIST_GET_LEN(space->chain) == 1); node = UT_LIST_GET_FIRST(space->chain); - if (node->n_pending > 0 - || node->n_pending_flushes > 0 - || node->being_extended) { - /* There are pending i/o's or flushes or the file is - currently being extended, sleep for a while and - retry */ - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(20000); - - goto retry; - - } else if (node->modification_counter > node->flush_counter) { - /* Flush the space */ - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(20000); - - fil_flush(id); - - goto retry; - - } else if (node->open) { - /* Close the file */ - - fil_node_close_file(node, fil_system); - } - /* Check that the old name in the space is right */ if (old_name_in) { @@ -3054,17 +2954,9 @@ retry: space, node, new_name, new_path); if (success) { - - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", - goto skip_second_rename; ); - success = os_file_rename( innodb_file_data_key, old_path, new_path); - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", -skip_second_rename: - success = FALSE; ); - if (!success) { /* We have to revert the changes we made to the tablespace memory cache */ @@ -3074,8 +2966,6 @@ skip_second_rename: } } - space->stop_ios = FALSE; - mutex_exit(&fil_system->mutex); #ifndef UNIV_HOTBACKUP diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 6230bd7a4d4..86b1c561349 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -249,10 +249,6 @@ struct fil_space_t { the space corresponds to a table in the InnoDB data dictionary; so we can print a warning of orphaned tablespaces */ - ibool stop_ios;/*!< TRUE if we want to rename the - .ibd file of tablespace and want to - stop temporarily posting of new i/o - requests on the file */ ibool stop_new_ops; /*!< we set this TRUE when we start deleting a single-table tablespace. diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 5ed7cbbc9b3..c1ddc83d852 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -1301,7 +1301,8 @@ os_file_create_simple_func( /* Use default security attributes and no template file. */ file = CreateFile( - (LPCTSTR) name, access, FILE_SHARE_READ, NULL, + (LPCTSTR) name, access, + FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, create_flag, attributes, NULL); if (file == INVALID_HANDLE_VALUE) { @@ -1460,7 +1461,7 @@ os_file_create_simple_no_error_handling_func( DWORD access; DWORD create_flag; DWORD attributes = 0; - DWORD share_mode = FILE_SHARE_READ; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; ut_a(name); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); @@ -1737,7 +1738,7 @@ os_file_create_func( #ifdef __WIN__ DWORD create_flag; - DWORD share_mode = FILE_SHARE_READ; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? TRUE : FALSE; From 619c277a6cc11b62e9d84449c0ee8ab39abc54f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 7 Jun 2018 14:11:55 +0300 Subject: [PATCH 7/8] Mariabackup: Make some globals static --- extra/mariabackup/backup_copy.cc | 1 + extra/mariabackup/xtrabackup.cc | 4 ++-- extra/mariabackup/xtrabackup.h | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 2ea34406a6b..1405df9bcc8 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -1205,6 +1205,7 @@ cleanup: +static bool backup_files(const char *from, bool prep_mode) { diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 91c7b5faf0e..e14e65d1679 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -202,9 +202,9 @@ my_bool opt_ssl_verify_server_cert; /* === metadata of backup === */ #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints" char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/ -lsn_t metadata_from_lsn; +static lsn_t metadata_from_lsn; lsn_t metadata_to_lsn; -lsn_t metadata_last_lsn; +static lsn_t metadata_last_lsn; static ds_file_t* dst_log_file; diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index c77485cb4de..2db5bc29b51 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -56,9 +56,7 @@ extern xb_page_bitmap *changed_page_bitmap; extern char *xtrabackup_incremental; extern my_bool xtrabackup_incremental_force_scan; -extern lsn_t metadata_from_lsn; extern lsn_t metadata_to_lsn; -extern lsn_t metadata_last_lsn; extern xb_stream_fmt_t xtrabackup_stream_fmt; extern ibool xtrabackup_stream; From dc9c555415b1d37b713839b9f8143a053f402c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 7 Jun 2018 14:29:35 +0300 Subject: [PATCH 8/8] MDEV-16367 mariabackup: error: failed to copy enough redo log log_copying_thread(): Keep copying redo log until the end has been reached. (Previously, we would stop copying as soon as the first batch of xtrabackup_copy_logfile() returned.) log_copying: Remove. Use log_copying_running instead. copy_logfile: Remove. Log copying will now only be invoked from 2 places: from xtrabackup_backup_func() for the initial batch, and from log_copying_thread() until all of the log has been read. Use the global variable metadata_to_lsn for determining if the final part of the log is being copied. xtrabackup_copy_log(): Add diagnostic messages for terminating the copying. These messages should be dead code, because log_group_read_log_seg() should be checking for the same. xtrabackup_copy_logfile(): Correct the retrying logic. If anything was successfully read, process the portion that was read. On failure, let the caller close dst_log_file. io_watching_thread(): Stop throttling during the last phase of copying the log (metadata_to_lsn!=0). The final copying of the log will now be performed in log_copying_thread(). stop_backup_threads(): Clean up the message about stopping the log copying thread. xtrabackup_backup_low(): Read metadata_to_lsn from the latest checkpoint header page, even if it is the first page. Let the log_copying_thread take care of copying all of the redo log. --- extra/mariabackup/xtrabackup.cc | 113 ++++++++++++++++---------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index e14e65d1679..65fddedc61a 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -172,7 +172,6 @@ typedef struct xb_filter_entry_struct xb_filter_entry_t; lsn_t checkpoint_lsn_start; lsn_t checkpoint_no_start; static lsn_t log_copy_scanned_lsn; -static bool log_copying; static bool log_copying_running; static bool io_watching_thread_running; @@ -2452,25 +2451,12 @@ skip: return(FALSE); } -/** How to copy a redo log segment in backup */ -enum copy_logfile { - /** Initial copying: copy at least one block */ - COPY_FIRST, - /** Tracking while copying data files */ - COPY_ONLINE, - /** Final copying: copy until the end of the log */ - COPY_LAST -}; - /** Copy redo log blocks to the data sink. -@param[in] copy how to copy the log -@param[in] start_lsn buffer start LSN -@param[in] end_lsn buffer end LSN -@return last scanned LSN (equals to last copied LSN if copy=COPY_LAST) +@param start_lsn buffer start LSN +@param end_lsn buffer end LSN +@return last scanned LSN @retval 0 on failure */ -static -lsn_t -xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) +static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn) { lsn_t scanned_lsn = start_lsn; const byte* log_block = log_sys->buf; @@ -2485,6 +2471,9 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) && scanned_checkpoint - checkpoint >= 0x80000000UL) { /* Garbage from a log buffer flush which was made before the most recent database recovery */ + msg("mariabackup: checkpoint wrap: " + LSN_PF ",%zx,%zx\n", + scanned_lsn, scanned_checkpoint, checkpoint); break; } @@ -2505,6 +2494,8 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) >= OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_TRL_SIZE || data_len <= LOG_BLOCK_HDR_SIZE) { /* We got a garbage block (abrupt end of the log). */ + msg("mariabackup: garbage block: " LSN_PF ",%zu\n", + scanned_lsn, data_len); break; } else { /* We got a partial block (abrupt end of the log). */ @@ -2524,7 +2515,7 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) log_sys->log.scanned_lsn = scanned_lsn; - end_lsn = copy == COPY_LAST + end_lsn = metadata_to_lsn ? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE) : scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1); @@ -2544,10 +2535,8 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) } /** Copy redo log until the current end of the log is reached -@param copy how to copy the log @return whether the operation failed */ -static bool -xtrabackup_copy_logfile(copy_logfile copy) +static bool xtrabackup_copy_logfile() { ut_a(dst_log_file != NULL); ut_ad(recv_sys != NULL); @@ -2560,33 +2549,29 @@ xtrabackup_copy_logfile(copy_logfile copy) start_lsn = ut_uint64_align_down(log_copy_scanned_lsn, OS_FILE_LOG_BLOCK_SIZE); - /* When copying the first or last part of the log, retry a few - times to ensure that all log up to the last checkpoint will be - read. */ do { end_lsn = start_lsn + RECV_SCAN_SIZE; xtrabackup_io_throttling(); log_mutex_enter(); - lsn_t lsn= start_lsn; - for(int retries= 0; retries < 100; retries++) { + for (int retries= 0; retries < 100; retries++) { if (log_group_read_log_seg(log_sys->buf, &log_sys->log, - &lsn, end_lsn)){ + &lsn, end_lsn) + || lsn != start_lsn) { break; } - msg("Retrying read of a redo log block"); + msg("Retrying read of log at LSN=" LSN_PF "\n", lsn); my_sleep(1000); } - start_lsn = xtrabackup_copy_log(copy, start_lsn, lsn); + start_lsn = (lsn == start_lsn) + ? 0 : xtrabackup_copy_log(start_lsn, lsn); log_mutex_exit(); if (!start_lsn) { - ds_close(dst_log_file); - dst_log_file = NULL; msg("mariabackup: Error: xtrabackup_copy_logfile()" " failed.\n"); return(true); @@ -2612,12 +2597,23 @@ static os_thread_ret_t log_copying_thread(void*) */ my_thread_init(); - do { + for (;;) { os_event_reset(log_copying_stop); os_event_wait_time_low(log_copying_stop, xtrabackup_log_copy_interval * 1000ULL, 0); - } while (log_copying && xtrabackup_copy_logfile(COPY_ONLINE)); + if (xtrabackup_copy_logfile()) { + break; + } + + log_mutex_enter(); + bool completed = metadata_to_lsn + && metadata_to_lsn < log_copy_scanned_lsn; + log_mutex_exit(); + if (completed) { + break; + } + } log_copying_running = false; my_thread_end(); @@ -2632,7 +2628,7 @@ static os_thread_ret_t io_watching_thread(void*) /* currently, for --backup only */ ut_a(xtrabackup_backup); - while (log_copying) { + while (log_copying_running && !metadata_to_lsn) { os_thread_sleep(1000000); /*1 sec*/ io_ticket = xtrabackup_throttle; os_event_set(wait_throttle); @@ -3662,16 +3658,16 @@ end: static void stop_backup_threads() { - log_copying = false; - - if (log_copying_stop) { + if (log_copying_stop && log_copying_running) { os_event_set(log_copying_stop); - msg("mariabackup: Stopping log copying thread.\n"); + fputs("mariabackup: Stopping log copying thread", stderr); + fflush(stderr); while (log_copying_running) { - msg("."); + putc('.', stderr); + fflush(stderr); os_thread_sleep(200000); /*0.2 sec*/ } - msg("\n"); + putc('\n', stderr); os_event_destroy(log_copying_stop); } @@ -3686,10 +3682,10 @@ static void stop_backup_threads() /** Implement the core of --backup @return whether the operation succeeded */ -static -bool -xtrabackup_backup_low() +static bool xtrabackup_backup_low() { + ut_ad(!metadata_to_lsn); + /* read the latest checkpoint lsn */ { ulint max_cp_field; @@ -3698,13 +3694,16 @@ xtrabackup_backup_low() if (recv_find_max_checkpoint(&max_cp_field) == DB_SUCCESS && log_sys->log.format != 0) { + if (max_cp_field == LOG_CHECKPOINT_1) { + log_group_header_read(&log_sys->log, + max_cp_field); + } metadata_to_lsn = mach_read_from_8( log_sys->checkpoint_buf + LOG_CHECKPOINT_LSN); msg("mariabackup: The latest check point" " (for incremental): '" LSN_PF "'\n", metadata_to_lsn); } else { - metadata_to_lsn = 0; msg("mariabackup: Error: recv_find_max_checkpoint() failed.\n"); } log_mutex_exit(); @@ -3712,11 +3711,7 @@ xtrabackup_backup_low() stop_backup_threads(); - if (!dst_log_file || xtrabackup_copy_logfile(COPY_LAST)) { - return false; - } - - if (ds_close(dst_log_file)) { + if (ds_close(dst_log_file) || !metadata_to_lsn) { dst_log_file = NULL; return false; } @@ -3795,6 +3790,7 @@ xtrabackup_backup_func() srv_read_only_mode = TRUE; srv_operation = SRV_OPERATION_BACKUP; + metadata_to_lsn = 0; if (xb_close_files) msg("mariabackup: warning: close-files specified. Use it " @@ -3805,7 +3801,12 @@ xtrabackup_backup_func() /* initialize components */ if(innodb_init_param()) { fail: + metadata_to_lsn = log_copying_running; stop_backup_threads(); + if (dst_log_file) { + ds_close(dst_log_file); + dst_log_file = NULL; + } if (fil_system) { innodb_shutdown(); } @@ -3814,7 +3815,6 @@ fail: xb_normalize_init_values(); - if (srv_file_flush_method_str == NULL) { /* These are the default options */ srv_file_flush_method = SRV_FSYNC; @@ -4058,9 +4058,7 @@ reread_log_header: goto log_write_fail; } - /* start flag */ - log_copying = TRUE; - + log_copying_running = true; /* start io throttle */ if(xtrabackup_throttle) { os_thread_id_t io_watching_thread_id; @@ -4078,6 +4076,8 @@ reread_log_header: if (err != DB_SUCCESS) { msg("mariabackup: error: xb_load_tablespaces() failed with" " error %s.\n", ut_strerr(err)); +fail_before_log_copying_thread_start: + log_copying_running = false; goto fail; } @@ -4085,11 +4085,10 @@ reread_log_header: log_copy_scanned_lsn = checkpoint_lsn_start; recv_sys->recovered_lsn = log_copy_scanned_lsn; - if (xtrabackup_copy_logfile(COPY_FIRST)) - goto fail; + if (xtrabackup_copy_logfile()) + goto fail_before_log_copying_thread_start; log_copying_stop = os_event_create(0); - log_copying_running = true; os_thread_create(log_copying_thread, NULL, &log_copying_thread_id); /* FLUSH CHANGED_PAGE_BITMAPS call */