From c95ba183d2528ea33a04210d9f46dabde36e46e3 Mon Sep 17 00:00:00 2001 From: Sophist <3001893+Sophist-UK@users.noreply.github.com> Date: Thu, 21 Dec 2023 20:08:32 +0000 Subject: [PATCH 01/12] Replace incorrect message `mariadb-safe` with correct `mariadbd-safe` --- scripts/mysql_install_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index bfca263478b..3bc4d63b7df 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -657,7 +657,7 @@ then then echo echo "You can start the MariaDB daemon with:" - echo "cd '$basedir' ; $bindir/mariadb-safe --datadir='$ldata'" + echo "cd '$basedir' ; $bindir/mariadbd-safe --datadir='$ldata'" echo echo "You can test the MariaDB daemon with mysql-test-run.pl" echo "cd '$basedir/@INSTALL_MYSQLTESTDIR@' ; perl mariadb-test-run.pl" From a6290a5bc5f3cba096854595c354d19d9267743d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2024 09:04:19 +0200 Subject: [PATCH 02/12] MDEV-33095 innodb_flush_method=O_DIRECT creates excessive errors on Solaris The directio(3C) function on Solaris is supported on NFS and UFS while the majority of users should be on ZFS, which is a copy-on-write file system that implements transparent compression and therefore cannot support unbuffered I/O. Let us remove the call to directio() and simply treat innodb_flush_method=O_DIRECT in the same way as the previous default value innodb_flush_method=fsync on Solaris. Also, let us remove some dead code around calls to os_file_set_nocache() on platforms where fcntl(2) is not usable with O_DIRECT. On IBM AIX, O_DIRECT is not documented for fcntl(2), only for open(2). --- cmake/os/AIX.cmake | 3 ++ cmake/os/SunOS.cmake | 4 +++ cmake/os/WindowsCache.cmake | 1 + config.h.cmake | 1 + configure.cmake | 1 + extra/mariabackup/fil_cur.cc | 2 ++ mysql-test/mariadb-test-run.pl | 2 +- storage/innobase/fil/fil0fil.cc | 12 ++++++-- storage/innobase/fil/fil0pagecompress.cc | 5 ---- storage/innobase/handler/ha_innodb.cc | 8 ++--- storage/innobase/include/os0file.h | 10 ++++--- storage/innobase/os/os0file.cc | 37 ++++++++++-------------- storage/innobase/row/row0merge.cc | 3 +- 13 files changed, 50 insertions(+), 39 deletions(-) diff --git a/cmake/os/AIX.cmake b/cmake/os/AIX.cmake index 299b79198c6..7513c4f42c2 100644 --- a/cmake/os/AIX.cmake +++ b/cmake/os/AIX.cmake @@ -34,5 +34,8 @@ ELSE() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -maix64 -pthread -mcmodel=large") ENDIF() +# fcntl(fd, F_SETFL, O_DIRECT) is not supported; O_DIRECT is an open(2) flag +SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "") + # make it WARN by default, not AUTO (that implies -Werror) SET(MYSQL_MAINTAINER_MODE "WARN" CACHE STRING "Enable MariaDB maintainer-specific warnings. One of: NO (warnings are disabled) WARN (warnings are enabled) ERR (warnings are errors) AUTO (warnings are errors in Debug only)") diff --git a/cmake/os/SunOS.cmake b/cmake/os/SunOS.cmake index 3a9d2dccb87..3d99d34789a 100644 --- a/cmake/os/SunOS.cmake +++ b/cmake/os/SunOS.cmake @@ -17,6 +17,10 @@ INCLUDE(CheckSymbolExists) INCLUDE(CheckCSourceRuns) INCLUDE(CheckCSourceCompiles) +# fcntl(fd, F_SETFL, O_DIRECT) is not supported, +# and directio(3C) would only work on UFS or NFS, not ZFS. +SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "") + # Enable 64 bit file offsets SET(_FILE_OFFSET_BITS 64) diff --git a/cmake/os/WindowsCache.cmake b/cmake/os/WindowsCache.cmake index c1048661aaa..ceb4262730f 100644 --- a/cmake/os/WindowsCache.cmake +++ b/cmake/os/WindowsCache.cmake @@ -44,6 +44,7 @@ SET(HAVE_EXECINFO_H CACHE INTERNAL "") SET(HAVE_FCHMOD CACHE INTERNAL "") SET(HAVE_FCNTL CACHE INTERNAL "") SET(HAVE_FCNTL_H 1 CACHE INTERNAL "") +SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "") SET(HAVE_FCNTL_NONBLOCK CACHE INTERNAL "") SET(HAVE_FDATASYNC CACHE INTERNAL "") SET(HAVE_DECL_FDATASYNC CACHE INTERNAL "") diff --git a/config.h.cmake b/config.h.cmake index 5075561d6e0..9b89d647543 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -30,6 +30,7 @@ #cmakedefine HAVE_DLFCN_H 1 #cmakedefine HAVE_EXECINFO_H 1 #cmakedefine HAVE_FCNTL_H 1 +#cmakedefine HAVE_FCNTL_DIRECT 1 #cmakedefine HAVE_FENV_H 1 #cmakedefine HAVE_FLOAT_H 1 #cmakedefine HAVE_FNMATCH_H 1 diff --git a/configure.cmake b/configure.cmake index cce8c4f611b..2d6f6183914 100644 --- a/configure.cmake +++ b/configure.cmake @@ -705,6 +705,7 @@ CHECK_SYMBOL_EXISTS(O_NONBLOCK "unistd.h;fcntl.h" HAVE_FCNTL_NONBLOCK) IF(NOT HAVE_FCNTL_NONBLOCK) SET(NO_FCNTL_NONBLOCK 1) ENDIF() +CHECK_SYMBOL_EXISTS(O_DIRECT "fcntl.h" HAVE_FCNTL_DIRECT) # # Test for how the C compiler does inline, if at all diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index 8820ce40c2b..ce36240fb70 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -199,11 +199,13 @@ xb_fil_cur_open( return(XB_FIL_CUR_SKIP); } +#ifdef HAVE_FCNTL_DIRECT if (srv_file_flush_method == SRV_O_DIRECT || srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) { os_file_set_nocache(cursor->file, node->name, "OPEN"); } +#endif posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 96fcd96c1fa..2750e10ecf5 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -4506,7 +4506,7 @@ sub extract_warning_lines ($$) { qr|InnoDB: io_setup\(\) failed with EAGAIN|, qr|io_uring_queue_init\(\) failed with|, qr|InnoDB: liburing disabled|, - qr/InnoDB: Failed to set (O_DIRECT|DIRECTIO_ON) on file/, + qr/InnoDB: Failed to set O_DIRECT on file/, qr|setrlimit could not change the size of core files to 'infinity';|, qr|feedback plugin: failed to retrieve the MAC address|, qr|Plugin 'FEEDBACK' init function returned error|, diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 34748199c0d..58598ec5581 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -357,8 +357,9 @@ static bool fil_node_open_file_low(fil_node_t *node) ut_ad(!node->is_open()); ut_ad(node->space->is_closing()); mysql_mutex_assert_owner(&fil_system.mutex); - ulint type; static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096, "compatibility"); +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT + ulint type; switch (FSP_FLAGS_GET_ZIP_SSIZE(node->space->flags)) { case 1: case 2: @@ -367,6 +368,9 @@ static bool fil_node_open_file_low(fil_node_t *node) default: type= OS_DATA_FILE; } +#else + constexpr auto type= OS_DATA_FILE; +#endif for (;;) { @@ -1937,9 +1941,10 @@ fil_ibd_create( mtr.commit(); log_write_up_to(mtr.commit_lsn(), true); - ulint type; static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096, "compatibility"); +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT + ulint type; switch (FSP_FLAGS_GET_ZIP_SSIZE(flags)) { case 1: case 2: @@ -1948,6 +1953,9 @@ fil_ibd_create( default: type = OS_DATA_FILE; } +#else + constexpr auto type = OS_DATA_FILE; +#endif file = os_file_create( innodb_data_file_key, path, diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 3d7c65eb01f..825df9cfe00 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -49,11 +49,6 @@ Updated 14/02/2015 #include "buf0lru.h" #include "ibuf0ibuf.h" #include "zlib.h" -#ifdef __linux__ -#include -#include -#include -#endif #include "row0mysql.h" #ifdef HAVE_LZ4 #include "lz4.h" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index af556d65695..5436956479a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4102,7 +4102,7 @@ static int innodb_init_params() data_mysql_default_charset_coll = (ulint) default_charset_info->number; -#ifndef _WIN32 +#ifdef HAVE_FCNTL_DIRECT if (srv_use_atomic_writes && my_may_have_atomic_write) { /* Force O_DIRECT on Unixes (on Windows writes are always @@ -4134,9 +4134,7 @@ static int innodb_init_params() } #endif -#ifndef _WIN32 - ut_ad(srv_file_flush_method <= SRV_O_DIRECT_NO_FSYNC); -#else +#ifdef _WIN32 switch (srv_file_flush_method) { case SRV_ALL_O_DIRECT_FSYNC + 1 /* "async_unbuffered"="unbuffered" */: srv_file_flush_method = SRV_ALL_O_DIRECT_FSYNC; @@ -4147,6 +4145,8 @@ static int innodb_init_params() default: ut_ad(srv_file_flush_method <= SRV_ALL_O_DIRECT_FSYNC); } +#else + ut_ad(srv_file_flush_method <= SRV_O_DIRECT_NO_FSYNC); #endif innodb_buffer_pool_size_init(); diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index c0db3b0d752..e0f398301e3 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -151,9 +151,11 @@ static const ulint OS_FILE_NORMAL = 62; /* @} */ /** Types for file create @{ */ -static const ulint OS_DATA_FILE = 100; -static const ulint OS_LOG_FILE = 101; -static const ulint OS_DATA_FILE_NO_O_DIRECT = 103; +static constexpr ulint OS_DATA_FILE = 100; +static constexpr ulint OS_LOG_FILE = 101; +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT +static constexpr ulint OS_DATA_FILE_NO_O_DIRECT = 103; +#endif /* @} */ /** Error codes from os_file_get_last_error @{ */ @@ -382,7 +384,7 @@ os_file_create_simple_no_error_handling_func( bool* success) MY_ATTRIBUTE((warn_unused_result)); -#ifdef _WIN32 +#ifndef HAVE_FCNTL_DIRECT #define os_file_set_nocache(fd, file_name, operation_name) do{}while(0) #else /** Tries to disable OS caching on an opened file descriptor. diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 7628939f2dd..014a9eb632f 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -970,7 +970,7 @@ os_file_create_simple_func( *success = false; int create_flag; - const char* mode_str = NULL; + const char* mode_str __attribute__((unused)); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); @@ -1046,12 +1046,14 @@ os_file_create_simple_func( } while (retry); +#ifdef HAVE_FCNTL_DIRECT /* This function is always called for data files, we should disable OS caching (O_DIRECT) here as we do in os_file_create_func(), so we open the same file in the same mode, see man page of open(2). */ if (!srv_read_only_mode && *success) { os_file_set_nocache(file, name, mode_str); } +#endif #ifndef _WIN32 if (!read_only @@ -1137,7 +1139,7 @@ os_file_create_func( ); int create_flag; - const char* mode_str = NULL; + const char* mode_str __attribute__((unused)); on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? true : false; @@ -1179,9 +1181,13 @@ os_file_create_func( return(OS_FILE_CLOSED); } +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT ut_a(type == OS_LOG_FILE || type == OS_DATA_FILE || type == OS_DATA_FILE_NO_O_DIRECT); +#else + ut_a(type == OS_LOG_FILE || type == OS_DATA_FILE); +#endif ut_a(purpose == OS_FILE_AIO || purpose == OS_FILE_NORMAL); @@ -1224,6 +1230,7 @@ os_file_create_func( } while (retry); +#ifdef HAVE_FCNTL_DIRECT /* We disable OS caching (O_DIRECT) only on data files */ if (!read_only && *success @@ -1231,6 +1238,7 @@ os_file_create_func( && type != OS_DATA_FILE_NO_O_DIRECT) { os_file_set_nocache(file, name, mode_str); } +#endif #ifndef _WIN32 if (!read_only @@ -2147,12 +2155,14 @@ os_file_create_func( } break; +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT case SRV_O_DIRECT_NO_FSYNC: case SRV_O_DIRECT: if (type != OS_DATA_FILE) { break; } /* fall through */ +#endif case SRV_ALL_O_DIRECT_FSYNC: /*Traditional Windows behavior, no buffering for any files.*/ if (type != OS_DATA_FILE_NO_O_DIRECT) { @@ -3037,17 +3047,14 @@ os_file_handle_error_cond_exit( return(false); } -#ifndef _WIN32 +#ifdef HAVE_FCNTL_DIRECT /** Tries to disable OS caching on an opened file descriptor. @param[in] fd file descriptor to alter @param[in] file_name file name, used in the diagnostic message @param[in] name "open" or "create"; used in the diagnostic message */ void -os_file_set_nocache( - int fd MY_ATTRIBUTE((unused)), - const char* file_name MY_ATTRIBUTE((unused)), - const char* operation_name MY_ATTRIBUTE((unused))) +os_file_set_nocache(int fd, const char *file_name, const char *operation_name) { const auto innodb_flush_method = srv_file_flush_method; switch (innodb_flush_method) { @@ -3058,18 +3065,6 @@ os_file_set_nocache( return; } - /* some versions of Solaris may not have DIRECTIO_ON */ -#if defined(__sun__) && defined(DIRECTIO_ON) - if (directio(fd, DIRECTIO_ON) == -1) { - int errno_save = errno; - - ib::error() - << "Failed to set DIRECTIO_ON on file " - << file_name << "; " << operation_name << ": " - << strerror(errno_save) << "," - " continuing anyway."; - } -#elif defined(O_DIRECT) if (fcntl(fd, F_SETFL, O_DIRECT) == -1) { int errno_save = errno; static bool warning_message_printed = false; @@ -3088,10 +3083,8 @@ os_file_set_nocache( << ", continuing anyway."; } } -#endif /* defined(__sun__) && defined(DIRECTIO_ON) */ } - -#endif /* _WIN32 */ +#endif /* HAVE_FCNTL_DIRECT */ /** Check if the file system supports sparse files. @param fh file handle diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 178c2836c7c..c5ac548c768 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -4193,13 +4193,14 @@ row_merge_file_create( merge_file->fd = row_merge_file_create_low(path); merge_file->offset = 0; merge_file->n_rec = 0; - +#ifdef HAVE_FCNTL_DIRECT if (merge_file->fd != OS_FILE_CLOSED) { if (srv_disable_sort_file_cache) { os_file_set_nocache(merge_file->fd, "row0merge.cc", "sort"); } } +#endif return(merge_file->fd); } From 82e8633420bbb0104a3242db4408ecef71932cac Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 19 Jan 2024 15:24:19 +1100 Subject: [PATCH 03/12] innodb: IO Error message missing space Noted by Susmeet Khaire - thanks. --- storage/innobase/os/os0file.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 014a9eb632f..3aede37909d 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3454,7 +3454,7 @@ static void write_io_callback(void *c) if (UNIV_UNLIKELY(cb->m_err != 0)) ib::info () << "IO Error: " << cb->m_err - << "during write of " + << " during write of " << cb->m_len << " bytes, for file " << request.node->name << "(" << cb->m_fh << "), returned " << cb->m_ret_len; From 16f2f8e5a7aa28c4bd4720f3281f8d1f1d06ba5b Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 17 Jan 2024 20:32:51 +0100 Subject: [PATCH 04/12] new CC 3.3 --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index 458a4396b44..26cef16b25f 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit 458a4396b443dcefedcf464067560078aa09d8b4 +Subproject commit 26cef16b25f01c1b159b0c73e3f6d9bcfcac90d5 From d34479dc664d4cbd4e9dad6b0f92d7c8e55595d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 19 Jan 2024 12:40:16 +0200 Subject: [PATCH 05/12] MDEV-33053 InnoDB LRU flushing does not run before running out of buffer pool buf_flush_LRU(): Display a warning if no pages could be evicted and no writes initiated. buf_pool_t::need_LRU_eviction(): Renamed from buf_pool_t::ran_out(). Check if the amount of free pages is smaller than innodb_lru_scan_depth instead of checking if it is 0. buf_flush_page_cleaner(): For the final LRU flush after a checkpoint flush, use a "budget" of innodb_io_capacity_max, like we do in the case when we are not in "furious" checkpoint flushing. Co-developed by: Debarun Banerjee Reviewed by: Debarun Banerjee Tested by: Matthias Leich --- storage/innobase/buf/buf0flu.cc | 71 +++++++++++++++++++++++------- storage/innobase/buf/buf0lru.cc | 31 +++++++++---- storage/innobase/include/buf0buf.h | 9 ++-- 3 files changed, 82 insertions(+), 29 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index f07ecedf535..ca9c71938c1 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1797,6 +1797,28 @@ ulint buf_flush_LRU(ulint max_n, bool evict) buf_pool.try_LRU_scan= true; pthread_cond_broadcast(&buf_pool.done_free); } + else if (!pages && !buf_pool.try_LRU_scan && + buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) + { + /* For example, with the minimum innodb_buffer_pool_size=5M and + the default innodb_page_size=16k there are only a little over 316 + pages in the buffer pool. The buffer pool can easily be exhausted + by a workload of some dozen concurrent connections. The system could + reach a deadlock like the following: + + (1) Many threads are waiting in buf_LRU_get_free_block() + for buf_pool.done_free. + (2) Some threads are waiting for a page latch which is held by + another thread that is waiting in buf_LRU_get_free_block(). + (3) This thread is the only one that could make progress, but + we fail to do so because all the pages that we scanned are + buffer-fixed or latched by some thread. */ + sql_print_warning("InnoDB: Could not free any blocks in the buffer pool!" + " %zu blocks are in use and %zu free." + " Consider increasing innodb_buffer_pool_size.", + UT_LIST_GET_LEN(buf_pool.LRU), + UT_LIST_GET_LEN(buf_pool.free)); + } return pages; } @@ -2287,6 +2309,16 @@ func_exit: goto func_exit; } +TPOOL_SUPPRESS_TSAN +bool buf_pool_t::need_LRU_eviction() const +{ + /* try_LRU_scan==false means that buf_LRU_get_free_block() is waiting + for buf_flush_page_cleaner() to evict some blocks */ + return UNIV_UNLIKELY(!try_LRU_scan || + (UT_LIST_GET_LEN(LRU) > BUF_LRU_MIN_LEN && + UT_LIST_GET_LEN(free) < srv_LRU_scan_depth / 2)); +} + /** page_cleaner thread tasked with flushing dirty pages from the buffer pools. As of now we'll have only one coordinator. */ static void buf_flush_page_cleaner() @@ -2319,21 +2351,24 @@ static void buf_flush_page_cleaner() } mysql_mutex_lock(&buf_pool.flush_list_mutex); - if (buf_pool.ran_out()) - goto no_wait; - else if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) - break; + if (!buf_pool.need_LRU_eviction()) + { + if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) + break; - if (buf_pool.page_cleaner_idle() && - (!UT_LIST_GET_LEN(buf_pool.flush_list) || - srv_max_dirty_pages_pct_lwm == 0.0)) - /* We are idle; wait for buf_pool.page_cleaner_wakeup() */ - my_cond_wait(&buf_pool.do_flush_list, - &buf_pool.flush_list_mutex.m_mutex); - else - my_cond_timedwait(&buf_pool.do_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - no_wait: + if (buf_pool.page_cleaner_idle() && + (!UT_LIST_GET_LEN(buf_pool.flush_list) || + srv_max_dirty_pages_pct_lwm == 0.0)) + { + buf_pool.LRU_warned.clear(std::memory_order_release); + /* We are idle; wait for buf_pool.page_cleaner_wakeup() */ + my_cond_wait(&buf_pool.do_flush_list, + &buf_pool.flush_list_mutex.m_mutex); + } + else + my_cond_timedwait(&buf_pool.do_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + } set_timespec(abstime, 1); lsn_limit= buf_flush_sync_lsn; @@ -2365,7 +2400,7 @@ static void buf_flush_page_cleaner() } while (false); - if (!buf_pool.ran_out()) + if (!buf_pool.need_LRU_eviction()) continue; mysql_mutex_lock(&buf_pool.flush_list_mutex); oldest_lsn= buf_pool.get_oldest_modification(0); @@ -2394,7 +2429,7 @@ static void buf_flush_page_cleaner() if (oldest_lsn >= soft_lsn_limit) buf_flush_async_lsn= soft_lsn_limit= 0; } - else if (buf_pool.ran_out()) + else if (buf_pool.need_LRU_eviction()) { buf_pool.page_cleaner_set_idle(false); buf_pool.n_flush_inc(); @@ -2509,9 +2544,11 @@ static void buf_flush_page_cleaner() MONITOR_FLUSH_ADAPTIVE_PAGES, n_flushed); } - else if (buf_flush_async_lsn <= oldest_lsn) + else if (buf_flush_async_lsn <= oldest_lsn && + !buf_pool.need_LRU_eviction()) goto check_oldest_and_set_idle; + n= srv_max_io_capacity; n= n >= n_flushed ? n - n_flushed : 0; goto LRU_flush; } diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index b80b99a9b12..ce37209f154 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -60,10 +60,6 @@ static constexpr ulint BUF_LRU_OLD_TOLERANCE = 20; frames in the buffer pool, we set this to TRUE */ static bool buf_lru_switched_on_innodb_mon = false; -/** True if diagnostic message about difficult to find free blocks -in the buffer bool has already printed. */ -static bool buf_lru_free_blocks_error_printed; - /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -408,6 +404,7 @@ got_mutex: buf_LRU_check_size_of_non_data_objects(); buf_block_t* block; + IF_DBUG(static bool buf_lru_free_blocks_error_printed,); DBUG_EXECUTE_IF("ib_lru_force_no_free_page", if (!buf_lru_free_blocks_error_printed) { n_iterations = 21; @@ -417,9 +414,25 @@ retry: /* If there is a block in the free list, take it */ if ((block = buf_LRU_get_free_only()) != nullptr) { got_block: + const ulint LRU_size = UT_LIST_GET_LEN(buf_pool.LRU); + const ulint available = UT_LIST_GET_LEN(buf_pool.free); + const ulint scan_depth = srv_LRU_scan_depth / 2; + ut_ad(LRU_size <= BUF_LRU_MIN_LEN || available >= scan_depth + || buf_pool.need_LRU_eviction()); + if (!have_mutex) { mysql_mutex_unlock(&buf_pool.mutex); } + + if (UNIV_UNLIKELY(available < scan_depth) + && LRU_size > BUF_LRU_MIN_LEN) { + mysql_mutex_lock(&buf_pool.flush_list_mutex); + if (!buf_pool.page_cleaner_active()) { + buf_pool.page_cleaner_wakeup(true); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + } + block->page.zip.clear(); return block; } @@ -445,10 +458,11 @@ got_block: if ((block = buf_LRU_get_free_only()) != nullptr) { goto got_block; } + const bool wake = buf_pool.need_LRU_eviction(); mysql_mutex_unlock(&buf_pool.mutex); mysql_mutex_lock(&buf_pool.flush_list_mutex); const auto n_flush = buf_pool.n_flush(); - if (!buf_pool.try_LRU_scan) { + if (wake && !buf_pool.page_cleaner_active()) { buf_pool.page_cleaner_wakeup(true); } mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -467,9 +481,10 @@ not_found: MONITOR_INC( MONITOR_LRU_GET_FREE_WAITS ); } - if (n_iterations == 21 && !buf_lru_free_blocks_error_printed - && srv_buf_pool_old_size == srv_buf_pool_size) { - buf_lru_free_blocks_error_printed = true; + if (n_iterations == 21 + && srv_buf_pool_old_size == srv_buf_pool_size + && buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) { + IF_DBUG(buf_lru_free_blocks_error_printed = true,); mysql_mutex_unlock(&buf_pool.mutex); ib::warn() << "Difficult to find free blocks in the buffer pool" " (" << n_iterations << " search iterations)! " diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 5626af17bb7..4986e83cfb9 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1488,10 +1488,8 @@ public: n_chunks_new / 4 * chunks->size; } - /** @return whether the buffer pool has run out */ - TPOOL_SUPPRESS_TSAN - bool ran_out() const - { return UNIV_UNLIKELY(!try_LRU_scan || !UT_LIST_GET_LEN(free)); } + /** @return whether the buffer pool is running low */ + bool need_LRU_eviction() const; /** @return whether the buffer pool is shrinking */ inline bool is_shrinking() const @@ -1811,6 +1809,9 @@ public: Set whenever the free list grows, along with a broadcast of done_free. Protected by buf_pool.mutex. */ Atomic_relaxed try_LRU_scan; + /** Whether we have warned to be running out of buffer pool */ + std::atomic_flag LRU_warned; + /* @} */ /** @name LRU replacement algorithm fields */ From 7e65e3027e7b55d64067ff0bef0dcbc9801fe967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 19 Jan 2024 12:40:32 +0200 Subject: [PATCH 06/12] MDEV-33275 buf_flush_LRU(): mysql_mutex_assert_owner(&buf_pool.mutex) failed In commit a55b951e6082a4ce9a1f2ed5ee176ea7dbbaf1f2 (MDEV-26827) an error was introduced in a rarely executed code path of the buf_flush_page_cleaner() thread. As a result, the function buf_flush_LRU() could be invoked while not holding buf_pool.mutex. Reviewed by: Debarun Banerjee --- storage/innobase/buf/buf0flu.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index ca9c71938c1..21ca288ec23 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -2547,6 +2547,8 @@ static void buf_flush_page_cleaner() else if (buf_flush_async_lsn <= oldest_lsn && !buf_pool.need_LRU_eviction()) goto check_oldest_and_set_idle; + else + mysql_mutex_lock(&buf_pool.mutex); n= srv_max_io_capacity; n= n >= n_flushed ? n - n_flushed : 0; From 21560bee9d6351b6f934a889bed123b548a49bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 19 Jan 2024 12:46:11 +0200 Subject: [PATCH 07/12] Revert "MDEV-32899 InnoDB is holding shared dict_sys.latch while waiting for FOREIGN KEY child table lock on DDL" This reverts commit 569da6a7bab034fc9768af88d8dbc2a8e2944764, commit 768a736174d6caf09df43e84b0c1b9ec52f1a301, and commit ba6bf7ad9e52c1dc31a22a619c17e1bb55b46d5d because of a regression that was filed as MDEV-33104. --- .../perfschema/r/sxlock_func,debug.rdiff | 22 ------- .../suite/perfschema/t/sxlock_func.test | 1 - storage/innobase/dict/dict0dict.cc | 37 ++++-------- storage/innobase/handler/ha_innodb.cc | 37 ++++++++---- storage/innobase/handler/handler0alter.cc | 11 +++- storage/innobase/include/dict0dict.h | 60 +++++++++---------- storage/innobase/include/lock0lock.h | 7 --- storage/innobase/lock/lock0lock.cc | 32 ---------- 8 files changed, 79 insertions(+), 128 deletions(-) delete mode 100644 mysql-test/suite/perfschema/r/sxlock_func,debug.rdiff diff --git a/mysql-test/suite/perfschema/r/sxlock_func,debug.rdiff b/mysql-test/suite/perfschema/r/sxlock_func,debug.rdiff deleted file mode 100644 index 0596810e553..00000000000 --- a/mysql-test/suite/perfschema/r/sxlock_func,debug.rdiff +++ /dev/null @@ -1,22 +0,0 @@ -@@ -7,7 +7,6 @@ - WHERE name LIKE 'wait/synch/rwlock/innodb/%' - AND name!='wait/synch/rwlock/innodb/btr_search_latch' ORDER BY name; - name --wait/synch/rwlock/innodb/dict_operation_lock - wait/synch/rwlock/innodb/fil_space_latch - wait/synch/rwlock/innodb/lock_latch - wait/synch/rwlock/innodb/trx_i_s_cache_lock -@@ -19,11 +18,13 @@ - select name from performance_schema.setup_instruments - where name like "wait/synch/sxlock/%" order by name; - name -+wait/synch/sxlock/innodb/dict_operation_lock - wait/synch/sxlock/innodb/index_tree_rw_lock - SELECT DISTINCT name FROM performance_schema.rwlock_instances - WHERE name LIKE 'wait/synch/sxlock/innodb/%' - ORDER BY name; - name -+wait/synch/sxlock/innodb/dict_operation_lock - wait/synch/sxlock/innodb/index_tree_rw_lock - create table t1(a int) engine=innodb; - begin; diff --git a/mysql-test/suite/perfschema/t/sxlock_func.test b/mysql-test/suite/perfschema/t/sxlock_func.test index 24d0e07ca41..c43adc849d2 100644 --- a/mysql-test/suite/perfschema/t/sxlock_func.test +++ b/mysql-test/suite/perfschema/t/sxlock_func.test @@ -5,7 +5,6 @@ --source include/not_embedded.inc --source include/have_perfschema.inc --source include/have_innodb.inc ---source include/maybe_debug.inc UPDATE performance_schema.setup_instruments SET enabled = 'NO', timed = 'YES'; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index d51ced003e8..a66bd0df510 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -958,12 +958,11 @@ void dict_sys_t::lock_wait(SRW_LOCK_ARGS(const char *file, unsigned line)) if (latch_ex_wait_start.compare_exchange_strong (old, now, std::memory_order_relaxed, std::memory_order_relaxed)) { -#ifdef UNIV_DEBUG - latch.x_lock(SRW_LOCK_ARGS(file, line)); -#else latch.wr_lock(SRW_LOCK_ARGS(file, line)); -#endif latch_ex_wait_start.store(0, std::memory_order_relaxed); + ut_ad(!latch_readers); + ut_ad(!latch_ex); + ut_d(latch_ex= pthread_self()); return; } @@ -978,39 +977,33 @@ void dict_sys_t::lock_wait(SRW_LOCK_ARGS(const char *file, unsigned line)) if (waited > threshold / 4) ib::warn() << "A long wait (" << waited << " seconds) was observed for dict_sys.latch"; -#ifdef UNIV_DEBUG - latch.x_lock(SRW_LOCK_ARGS(file, line)); -#else latch.wr_lock(SRW_LOCK_ARGS(file, line)); -#endif + ut_ad(!latch_readers); + ut_ad(!latch_ex); + ut_d(latch_ex= pthread_self()); } #ifdef UNIV_PFS_RWLOCK ATTRIBUTE_NOINLINE void dict_sys_t::unlock() { -# ifdef UNIV_DEBUG - latch.x_unlock(); -# else + ut_ad(latch_ex == pthread_self()); + ut_ad(!latch_readers); + ut_d(latch_ex= 0); latch.wr_unlock(); -# endif } ATTRIBUTE_NOINLINE void dict_sys_t::freeze(const char *file, unsigned line) { -# ifdef UNIV_DEBUG - latch.s_lock(file, line); -# else latch.rd_lock(file, line); -# endif + ut_ad(!latch_ex); + ut_d(latch_readers++); } ATTRIBUTE_NOINLINE void dict_sys_t::unfreeze() { -# ifdef UNIV_DEBUG - latch.s_unlock(); -# else + ut_ad(!latch_ex); + ut_ad(latch_readers--); latch.rd_unlock(); -# endif } #endif /* UNIV_PFS_RWLOCK */ @@ -4544,11 +4537,7 @@ void dict_sys_t::close() temp_id_hash.free(); unlock(); -#ifdef UNIV_DEBUG - latch.free(); -#else latch.destroy(); -#endif mysql_mutex_destroy(&dict_foreign_err_mutex); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5436956479a..98f3fa0344d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -598,13 +598,7 @@ static PSI_rwlock_info all_innodb_rwlocks[] = # ifdef BTR_CUR_HASH_ADAPT { &btr_search_latch_key, "btr_search_latch", 0 }, # endif - { &dict_operation_lock_key, "dict_operation_lock", -# ifdef UNIV_DEBUG - PSI_RWLOCK_FLAG_SX -# else - 0 -# endif - }, + { &dict_operation_lock_key, "dict_operation_lock", 0 }, { &fil_space_latch_key, "fil_space_latch", 0 }, { &trx_i_s_cache_lock_key, "trx_i_s_cache_lock", 0 }, { &trx_purge_latch_key, "trx_purge_latch", 0 }, @@ -13546,7 +13540,14 @@ int ha_innobase::delete_table(const char *name) /* FOREIGN KEY constraints cannot exist on partitioned tables. */; #endif else - err= lock_table_children(table, trx); + { + dict_sys.freeze(SRW_LOCK_CALL); + for (const dict_foreign_t* f : table->referenced_set) + if (dict_table_t* child= f->foreign_table) + if ((err= lock_table_for_trx(child, trx, LOCK_X)) != DB_SUCCESS) + break; + dict_sys.unfreeze(); + } } dict_table_t *table_stats= nullptr, *index_stats= nullptr; @@ -13944,7 +13945,14 @@ int ha_innobase::truncate() dict_table_t *table_stats = nullptr, *index_stats = nullptr; MDL_ticket *mdl_table = nullptr, *mdl_index = nullptr; - dberr_t error= lock_table_children(ib_table, trx); + dberr_t error= DB_SUCCESS; + + dict_sys.freeze(SRW_LOCK_CALL); + for (const dict_foreign_t *f : ib_table->referenced_set) + if (dict_table_t *child= f->foreign_table) + if ((error= lock_table_for_trx(child, trx, LOCK_X)) != DB_SUCCESS) + break; + dict_sys.unfreeze(); if (error == DB_SUCCESS) error= lock_table_for_trx(ib_table, trx, LOCK_X); @@ -14135,7 +14143,16 @@ ha_innobase::rename_table( /* There is no need to lock any FOREIGN KEY child tables. */ } else if (dict_table_t *table = dict_table_open_on_name( norm_from, false, DICT_ERR_IGNORE_FK_NOKEY)) { - error = lock_table_children(table, trx); + dict_sys.freeze(SRW_LOCK_CALL); + for (const dict_foreign_t* f : table->referenced_set) { + if (dict_table_t* child = f->foreign_table) { + error = lock_table_for_trx(child, trx, LOCK_X); + if (error != DB_SUCCESS) { + break; + } + } + } + dict_sys.unfreeze(); if (error == DB_SUCCESS) { error = lock_table_for_trx(table, trx, LOCK_X); } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index dc5c1f71925..3b892434a67 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -11203,7 +11203,16 @@ ha_innobase::commit_inplace_alter_table( fts_optimize_remove_table(ctx->old_table); } - error = lock_table_children(ctx->old_table, trx); + dict_sys.freeze(SRW_LOCK_CALL); + for (auto f : ctx->old_table->referenced_set) { + if (dict_table_t* child = f->foreign_table) { + error = lock_table_for_trx(child, trx, LOCK_X); + if (error != DB_SUCCESS) { + break; + } + } + } + dict_sys.unfreeze(); if (ctx->new_table->fts) { ut_ad(!ctx->new_table->fts->add_wq); diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index e4fc58007cf..895743be84b 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -1316,14 +1316,14 @@ class dict_sys_t /** The my_hrtime_coarse().val of the oldest lock_wait() start, or 0 */ std::atomic latch_ex_wait_start; -#ifdef UNIV_DEBUG - typedef index_lock dict_lock; -#else - typedef srw_lock dict_lock; -#endif - /** the rw-latch protecting the data dictionary cache */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) dict_lock latch; + alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_lock latch; +#ifdef UNIV_DEBUG + /** whether latch is being held in exclusive mode (by any thread) */ + Atomic_relaxed latch_ex; + /** number of S-latch holders */ + Atomic_counter latch_readers; +#endif public: /** Indexes of SYS_TABLE[] */ enum @@ -1491,12 +1491,15 @@ public: } #ifdef UNIV_DEBUG - /** @return whether the current thread is holding the latch */ - bool frozen() const { return latch.have_any(); } - /** @return whether the current thread is holding a shared latch */ - bool frozen_not_locked() const { return latch.have_s(); } + /** @return whether any thread (not necessarily the current thread) + is holding the latch; that is, this check may return false + positives */ + bool frozen() const { return latch_readers || latch_ex; } + /** @return whether any thread (not necessarily the current thread) + is holding a shared latch */ + bool frozen_not_locked() const { return latch_readers; } /** @return whether the current thread holds the exclusive latch */ - bool locked() const { return latch.have_x(); } + bool locked() const { return latch_ex == pthread_self(); } #endif private: /** Acquire the exclusive latch */ @@ -1511,12 +1514,13 @@ public: /** Exclusively lock the dictionary cache. */ void lock(SRW_LOCK_ARGS(const char *file, unsigned line)) { -#ifdef UNIV_DEBUG - ut_ad(!latch.have_any()); - if (!latch.x_lock_try()) -#else - if (!latch.wr_lock_try()) -#endif + if (latch.wr_lock_try()) + { + ut_ad(!latch_readers); + ut_ad(!latch_ex); + ut_d(latch_ex= pthread_self()); + } + else lock_wait(SRW_LOCK_ARGS(file, line)); } @@ -1531,30 +1535,24 @@ public: /** Unlock the data dictionary cache. */ void unlock() { -# ifdef UNIV_DEBUG - latch.x_unlock(); -# else + ut_ad(latch_ex == pthread_self()); + ut_ad(!latch_readers); + ut_d(latch_ex= 0); latch.wr_unlock(); -# endif } /** Acquire a shared lock on the dictionary cache. */ void freeze() { -# ifdef UNIV_DEBUG - ut_ad(!latch.have_any()); - latch.s_lock(); -# else latch.rd_lock(); -# endif + ut_ad(!latch_ex); + ut_d(latch_readers++); } /** Release a shared lock on the dictionary cache. */ void unfreeze() { -# ifdef UNIV_DEBUG - latch.s_unlock(); -# else + ut_ad(!latch_ex); + ut_ad(latch_readers--); latch.rd_unlock(); -# endif } #endif diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index 65537859924..59ee7f551b4 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -438,13 +438,6 @@ dberr_t lock_table_for_trx(dict_table_t *table, trx_t *trx, lock_mode mode, bool no_wait= false) MY_ATTRIBUTE((nonnull, warn_unused_result)); -/** Lock the child tables of a table. -@param table parent table -@param trx transaction -@return error code */ -dberr_t lock_table_children(dict_table_t *table, trx_t *trx) - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Exclusively lock the data dictionary tables. @param trx dictionary transaction @return error code diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index c9072998e66..df51ceb16d8 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -3940,8 +3940,6 @@ static void lock_table_dequeue(lock_t *in_lock, bool owns_wait_mutex) dberr_t lock_table_for_trx(dict_table_t *table, trx_t *trx, lock_mode mode, bool no_wait) { - ut_ad(!dict_sys.frozen()); - mem_heap_t *heap= mem_heap_create(512); sel_node_t *node= sel_node_create(heap); que_thr_t *thr= pars_complete_graph_for_exec(node, trx, heap, nullptr); @@ -3978,36 +3976,6 @@ run_again: return err; } -/** Lock the child tables of a table. -@param table parent table -@param trx transaction -@return error code */ -dberr_t lock_table_children(dict_table_t *table, trx_t *trx) -{ - dict_sys.freeze(SRW_LOCK_CALL); - std::vector children; - - for (auto f : table->referenced_set) - if (dict_table_t *child= f->foreign_table) - { - child->acquire(); - children.emplace_back(child); - } - dict_sys.unfreeze(); - - dberr_t err= DB_SUCCESS; - - for (auto child : children) - if ((err= lock_table_for_trx(child, trx, LOCK_X)) != DB_SUCCESS) - break; - - for (auto child : children) - child->release(); - - return err; -} - - /** Exclusively lock the data dictionary tables. @param trx dictionary transaction @return error code From 7573fe8b07c52fdd45eac3d4f0f764f1c61ca41e Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 19 Jan 2024 15:00:13 +0530 Subject: [PATCH 08/12] MDEV-32968 InnoDB fails to restore tablespace first page from doublewrite buffer when page is empty recv_dblwr_t::find_first_page(): Free the allocated memory to read the first 3 pages from tablespace. innodb.doublewrite: Added sleep to ensure page cleaner thread wake up from my_cond_wait --- mysql-test/suite/innodb/t/doublewrite.test | 3 +++ storage/innobase/log/log0recv.cc | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index 541e4d23a19..7e38851facb 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -39,6 +39,9 @@ commit work; SET GLOBAL innodb_fast_shutdown = 0; let $shutdown_timeout=; --source include/restart_mysqld.inc +# Ensure that buf_flush_page_cleaner() has woken up from its +# first my_cond_timedwait() and gone idle. +sleep 1; --source ../include/no_checkpoint_start.inc connect (dml,localhost,root,,); XA START 'x'; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 52649419ec7..ffe35e9dbc7 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3871,16 +3871,20 @@ uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) for (const page_t *page : pages) { uint32_t space_id= page_get_space_id(page); + byte *read_page= nullptr; if (page_get_page_no(page) > 0 || space_id == 0) + { next_page: + aligned_free(read_page); continue; + } uint32_t flags= mach_read_from_4( FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); page_id_t page_id(space_id, 0); size_t page_size= fil_space_t::physical_size(flags); if (file_size < 4 * page_size) goto next_page; - byte *read_page= + read_page= static_cast(aligned_malloc(3 * page_size, page_size)); /* Read 3 pages from the file and match the space id with the space id which is stored in @@ -3892,7 +3896,10 @@ next_page: { byte *cur_page= read_page + j * page_size; if (buf_is_zeroes(span(cur_page, page_size))) - return 0; + { + space_id= 0; + goto early_exit; + } if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || memcmp(cur_page + FIL_PAGE_SPACE_ID, page + FIL_PAGE_SPACE_ID, 4) || @@ -3900,7 +3907,11 @@ next_page: goto next_page; } if (!restore_first_page(space_id, name, file)) + { +early_exit: + aligned_free(read_page); return space_id; + } break; } } From 3a33ae86010051a1022c392f2936b5fc9f3ab19e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 12 Jan 2024 16:57:37 +0100 Subject: [PATCH 09/12] MDEV-33091 pcre2 headers aren't found on Solaris use pkg-config to find pcre2, if possible rename PCRE_INCLUDES to use PKG_CHECK_MODULES naming, PCRE_INCLUDE_DIRS --- client/CMakeLists.txt | 2 +- cmake/pcre.cmake | 34 +++++++++++++--------- cmake/plugin.cmake | 2 +- extra/mariabackup/CMakeLists.txt | 2 +- libmysqld/CMakeLists.txt | 2 +- libmysqld/examples/CMakeLists.txt | 2 +- plugin/feedback/CMakeLists.txt | 2 +- plugin/qc_info/CMakeLists.txt | 2 +- sql/CMakeLists.txt | 2 +- storage/perfschema/CMakeLists.txt | 2 +- storage/perfschema/unittest/CMakeLists.txt | 2 +- unittest/embedded/CMakeLists.txt | 2 +- 12 files changed, 32 insertions(+), 24 deletions(-) diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index 55fd02b23c3..4dfce2478cd 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -16,7 +16,7 @@ INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/mysys_ssl ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} diff --git a/cmake/pcre.cmake b/cmake/pcre.cmake index 65dc2ae28f6..c2eb26074ac 100644 --- a/cmake/pcre.cmake +++ b/cmake/pcre.cmake @@ -1,4 +1,3 @@ -INCLUDE (CheckCSourceRuns) INCLUDE (ExternalProject) SET(WITH_PCRE "auto" CACHE STRING @@ -6,7 +5,8 @@ SET(WITH_PCRE "auto" CACHE STRING MACRO(BUNDLE_PCRE2) SET(dir "${CMAKE_BINARY_DIR}/extra/pcre2") - SET(PCRE_INCLUDES ${dir}/src/pcre2-build ${dir}/src/pcre2/src) + SET(PCRE_INCLUDE_DIRS ${dir}/src/pcre2-build ${dir}/src/pcre2/src) + MESSAGE(STATUS "Will download and bundle pcre2") SET(byproducts) FOREACH(lib pcre2-posix pcre2-8) ADD_LIBRARY(${lib} STATIC IMPORTED GLOBAL) @@ -76,18 +76,26 @@ SET_TARGET_PROPERTIES(pcre2 PROPERTIES EXCLUDE_FROM_ALL TRUE) ENDMACRO() MACRO (CHECK_PCRE) - IF(WITH_PCRE STREQUAL "system" OR WITH_PCRE STREQUAL "auto") - CHECK_LIBRARY_EXISTS(pcre2-8 pcre2_match_8 "" HAVE_PCRE2) - ENDIF() - IF(NOT HAVE_PCRE2 OR WITH_PCRE STREQUAL "bundled") - IF (WITH_PCRE STREQUAL "system") - MESSAGE(FATAL_ERROR "system pcre2-8 library is not found or unusable") + IF (NOT TARGET pcre2 AND NOT PCRE_FOUND) + IF(WITH_PCRE STREQUAL "system" OR WITH_PCRE STREQUAL "auto") + FIND_PACKAGE(PkgConfig QUIET) + PKG_CHECK_MODULES(PCRE libpcre2-8) + # in case pkg-config or libpcre2-8.pc is not installed: + IF(NOT PCRE_FOUND) + UNSET(PCRE_FOUND CACHE) + CHECK_LIBRARY_EXISTS(pcre2-8 pcre2_match_8 "" PCRE_FOUND) + ENDIF() ENDIF() - BUNDLE_PCRE2() - ELSE() - CHECK_LIBRARY_EXISTS(pcre2-posix PCRE2regcomp "" NEEDS_PCRE2_DEBIAN_HACK) - IF(NEEDS_PCRE2_DEBIAN_HACK) - SET(PCRE2_DEBIAN_HACK "-Dregcomp=PCRE2regcomp -Dregexec=PCRE2regexec -Dregerror=PCRE2regerror -Dregfree=PCRE2regfree") + IF(NOT PCRE_FOUND OR WITH_PCRE STREQUAL "bundled") + IF (WITH_PCRE STREQUAL "system") + MESSAGE(FATAL_ERROR "system pcre2-8 library is not found or unusable") + ENDIF() + BUNDLE_PCRE2() + ELSE() + CHECK_LIBRARY_EXISTS(pcre2-posix PCRE2regcomp "" NEEDS_PCRE2_DEBIAN_HACK) + IF(NEEDS_PCRE2_DEBIAN_HACK) + SET(PCRE2_DEBIAN_HACK "-Dregcomp=PCRE2regcomp -Dregexec=PCRE2regexec -Dregerror=PCRE2regerror -Dregfree=PCRE2regfree") + ENDIF() ENDIF() ENDIF() ENDMACRO() diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index 813d8ef6e42..6efd40fd1bd 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -44,7 +44,7 @@ MACRO(MYSQL_ADD_PLUGIN) # Add common include directories INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR}) diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index a7a35c58ac3..4dfef82d3d5 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -36,7 +36,7 @@ INCLUDE_DIRECTORIES( ) IF(NOT HAVE_SYSTEM_REGEX) - INCLUDE_DIRECTORIES(${PCRE_INCLUDES}) + INCLUDE_DIRECTORIES(${PCRE_INCLUDE_DIRS}) ADD_DEFINITIONS(${PCRE2_DEBIAN_HACK}) ENDIF() diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index b414903f705..ff6e2cbf36d 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -23,7 +23,7 @@ ${CMAKE_SOURCE_DIR}/libmysqld ${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/tpool ${CMAKE_BINARY_DIR}/sql -${PCRE_INCLUDES} +${PCRE_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} ${SSL_INTERNAL_INCLUDE_DIRS} diff --git a/libmysqld/examples/CMakeLists.txt b/libmysqld/examples/CMakeLists.txt index 2a10def8e2e..d6646a128ca 100644 --- a/libmysqld/examples/CMakeLists.txt +++ b/libmysqld/examples/CMakeLists.txt @@ -15,7 +15,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/libmysqld/include - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/sql ${MY_READLINE_INCLUDE_DIR} ) diff --git a/plugin/feedback/CMakeLists.txt b/plugin/feedback/CMakeLists.txt index 2103250e5a6..fc35cbadc31 100644 --- a/plugin/feedback/CMakeLists.txt +++ b/plugin/feedback/CMakeLists.txt @@ -1,5 +1,5 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS}) SET(FEEDBACK_SOURCES feedback.cc sender_thread.cc diff --git a/plugin/qc_info/CMakeLists.txt b/plugin/qc_info/CMakeLists.txt index b8c5f926cff..329f49c1fc9 100644 --- a/plugin/qc_info/CMakeLists.txt +++ b/plugin/qc_info/CMakeLists.txt @@ -1,4 +1,4 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql - ${PCRE_INCLUDES}) + ${PCRE_INCLUDE_DIRS}) MYSQL_ADD_PLUGIN(QUERY_CACHE_INFO qc_info.cc RECOMPILE_FOR_EMBEDDED) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 557fd5eb506..f1c1c65310b 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -52,7 +52,7 @@ ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql -${PCRE_INCLUDES} +${PCRE_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} ${CMAKE_BINARY_DIR}/sql diff --git a/storage/perfschema/CMakeLists.txt b/storage/perfschema/CMakeLists.txt index b4f5e96b607..e703e43fe50 100644 --- a/storage/perfschema/CMakeLists.txt +++ b/storage/perfschema/CMakeLists.txt @@ -24,7 +24,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql ${CMAKE_BINARY_DIR}/sql ${CMAKE_CURRENT_BINARY_DIR} - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS}) ADD_DEFINITIONS(-DMYSQL_SERVER) diff --git a/storage/perfschema/unittest/CMakeLists.txt b/storage/perfschema/unittest/CMakeLists.txt index 2a22990f807..600795c78fc 100644 --- a/storage/perfschema/unittest/CMakeLists.txt +++ b/storage/perfschema/unittest/CMakeLists.txt @@ -22,7 +22,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/include/mysql - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/sql ${SSL_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/unittest/mytap diff --git a/unittest/embedded/CMakeLists.txt b/unittest/embedded/CMakeLists.txt index cf48550c377..428bb811de6 100644 --- a/unittest/embedded/CMakeLists.txt +++ b/unittest/embedded/CMakeLists.txt @@ -1,7 +1,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/libmysqld/include - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/sql ${MY_READLINE_INCLUDE_DIR} ) From 0c23f84d8dcb61c25849c07149933245333a32d9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Dec 2023 10:15:55 +1100 Subject: [PATCH 10/12] MDEV-32983 cosmetic improvement on path separator near ib_buffer_pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mix of path separators looks odd. InnoDB: Loading buffer pool(s) from C:\xampp\mysql\data/ib_buffer_pool This was changed in cf552f5886968fc022122960d3a9274ce9f27819 Both forward slashes and backward slashes work on Windows. We do not use \\?\ names. So we improve the consistent look of it so it doesn't look like a bug. Normalize, in this case, the path separator to \ for making the filename. Reported thanks to Github user @celestinoxp. Closes: https://github.com/ApacheFriends/xampp-build/issues/33 Reviewed by: Marko Mäkelä and Vladislav Vaintroub --- storage/innobase/buf/buf0dump.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index f142263a0e4..e9d544f9616 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -180,7 +180,7 @@ static void buf_dump_generate_path(char *path, size_t path_size) char buf[FN_REFLEN]; mysql_mutex_lock(&LOCK_global_system_variables); - snprintf(buf, sizeof buf, "%s/%s", get_buf_dump_dir(), + snprintf(buf, sizeof buf, "%s" FN_ROOTDIR "%s", get_buf_dump_dir(), srv_buf_dump_filename); mysql_mutex_unlock(&LOCK_global_system_variables); @@ -214,7 +214,7 @@ static void buf_dump_generate_path(char *path, size_t path_size) format = "%s%s"; break; default: - format = "%s/%s"; + format = "%s" FN_ROOTDIR "%s"; } snprintf(path, path_size, format, From e237925963eeff72809a7b9aafb4bd838c457064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 22 Jan 2024 08:19:00 +0200 Subject: [PATCH 11/12] MDEV-33031 test fixup for HAVE_PERFSCHEMA=NO --- storage/spider/mysql-test/spider/bugfix/t/perfschema.opt | 2 +- storage/spider/mysql-test/spider/bugfix/t/perfschema.test | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt b/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt index 611d08f0c78..d2ed32ddf34 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt +++ b/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt @@ -1 +1 @@ ---performance-schema +--loose-performance-schema diff --git a/storage/spider/mysql-test/spider/bugfix/t/perfschema.test b/storage/spider/mysql-test/spider/bugfix/t/perfschema.test index 2f1a961a513..9346d2b5aa3 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/perfschema.test +++ b/storage/spider/mysql-test/spider/bugfix/t/perfschema.test @@ -1,3 +1,4 @@ +source include/have_perfschema.inc; disable_query_log; source ../../include/init_spider.inc; enable_query_log; From 495e7f1b3ddc53a9f588a5e0b1f9ca2f2d82e43d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 22 Jan 2024 08:24:08 +0200 Subject: [PATCH 12/12] MDEV-33053 fixup: Correct a condition before a message --- storage/innobase/buf/buf0flu.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 21ca288ec23..4f7fb316b88 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1798,7 +1798,7 @@ ulint buf_flush_LRU(ulint max_n, bool evict) pthread_cond_broadcast(&buf_pool.done_free); } else if (!pages && !buf_pool.try_LRU_scan && - buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) + !buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) { /* For example, with the minimum innodb_buffer_pool_size=5M and the default innodb_page_size=16k there are only a little over 316