From 78c9a12c8fc25a6032e6c04b83171aafeebc53de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 22 Nov 2023 16:54:41 +0200 Subject: [PATCH] MDEV-32861 InnoDB hangs when running out of I/O slots When the constant OS_AIO_N_PENDING_IOS_PER_THREAD is changed from 256 to 1 and the server is run with the minimum parameters innodb_read_io_threads=1 and innodb_write_io_threads=2, two hangs were observed. tpool::cache::put(T*): Ensure that get() in io_slots::acquire() will be woken up when the cache previously was empty. buf_pool_t::io_buf_t::reserve(): Schedule a possibly partial doublewrite batch so that os_aio_wait_until_no_pending_writes() has a chance of returning. Add a Boolean parameter and pass wait_for_reads=false inside buf_page_decrypt_after_read(), because those calls will be executed inside a read completion callback, and therefore os_aio_wait_until_no_pending_reads() would block indefinitely. --- storage/innobase/buf/buf0buf.cc | 11 +++++++---- storage/innobase/buf/buf0flu.cc | 2 +- storage/innobase/include/buf0buf.h | 5 +++-- tpool/tpool_structs.h | 3 ++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index f762cf65a01..c0dece69cf8 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -414,7 +414,7 @@ static bool buf_page_decrypt_after_read(buf_page_t *bpage, if (node.space->purpose == FIL_TYPE_TEMPORARY && innodb_encrypt_temporary_tables) { - slot = buf_pool.io_buf_reserve(); + slot = buf_pool.io_buf_reserve(false); ut_a(slot); slot->allocate(); @@ -444,7 +444,7 @@ decompress: return false; } - slot = buf_pool.io_buf_reserve(); + slot = buf_pool.io_buf_reserve(false); slot->allocate(); decompress_with_slot: @@ -471,7 +471,7 @@ decrypt_failed: return false; } - slot = buf_pool.io_buf_reserve(); + slot = buf_pool.io_buf_reserve(false); slot->allocate(); ut_d(fil_page_type_validate(node.space, dst_frame)); @@ -1513,14 +1513,17 @@ void buf_pool_t::io_buf_t::close() n_slots= 0; } -buf_tmp_buffer_t *buf_pool_t::io_buf_t::reserve() +buf_tmp_buffer_t *buf_pool_t::io_buf_t::reserve(bool wait_for_reads) { for (;;) { for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++) if (s->acquire()) return s; + buf_dblwr.flush_buffered_writes(); os_aio_wait_until_no_pending_writes(); + if (!wait_for_reads) + continue; for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++) if (s->acquire()) return s; diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 77008312727..186fb89ddce 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -702,7 +702,7 @@ static byte *buf_page_encrypt(fil_space_t* space, buf_page_t* bpage, byte* s, ut_ad(!bpage->zip_size() || !page_compressed); /* Find free slot from temporary memory array */ - buf_tmp_buffer_t *slot= buf_pool.io_buf_reserve(); + buf_tmp_buffer_t *slot= buf_pool.io_buf_reserve(true); ut_a(slot); slot->allocate(); slot->out_buf= NULL; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index def7641db1d..0e1a4c27f35 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -2030,7 +2030,8 @@ public: a delete-buffering operation is pending. Protected by mutex. */ buf_page_t watch[innodb_purge_threads_MAX + 1]; /** Reserve a buffer. */ - buf_tmp_buffer_t *io_buf_reserve() { return io_buf.reserve(); } + buf_tmp_buffer_t *io_buf_reserve(bool wait_for_reads) + { return io_buf.reserve(wait_for_reads); } /** @return whether any I/O is pending */ bool any_io_pending() @@ -2083,7 +2084,7 @@ private: void close(); /** Reserve a buffer */ - buf_tmp_buffer_t *reserve(); + buf_tmp_buffer_t *reserve(bool wait_for_reads); } io_buf; /** whether resize() is in the critical path */ diff --git a/tpool/tpool_structs.h b/tpool/tpool_structs.h index b49204f2d75..96c250ebd59 100644 --- a/tpool/tpool_structs.h +++ b/tpool/tpool_structs.h @@ -138,12 +138,13 @@ public: { std::unique_lock lk(m_mtx); assert(!is_full()); + const bool was_empty= is_empty(); // put element to the logical end of the array m_cache[--m_pos] = ele; /* Notify waiters when the cache becomes not empty, or when it becomes full */ - if (m_pos == 1 || (m_waiters && is_full())) + if (was_empty || (is_full() && m_waiters)) m_cv.notify_all(); }