From 8d210fc2aa7ca08450055e2db3249b0f18e61bdb Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 9 Aug 2023 15:47:06 +0300 Subject: [PATCH 1/2] MDEV-31877: ASAN errors in Exec_time_tracker::get_cycles with innodb slow log verbosity Remove redundant delete_explain_query() calls in sp_instr_set::exec_core(), sp_instr_set_row_field::exec_core(), sp_instr_set_row_field_by_name::exec_core(). These calls are made before the SP instruction's tables are "closed" by close_thread_tables() call. When we call close_thread_tables() after that, we no longer can collect engine's counter variables, as they use the data structures that are located in the Explain Data Structures. Also, these delete_explain_query() calls are redundant, as sp_lex_keeper::reset_lex_and_exec_core() has another delete_explain_query() call, which is located in the right location after the close_thread_tables() call. --- mysql-test/main/sp.result | 12 ++++++++++++ mysql-test/main/sp.test | 16 ++++++++++++++++ sql/sp_head.cc | 3 --- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/mysql-test/main/sp.result b/mysql-test/main/sp.result index 88df800b0e0..74734a9b7ed 100644 --- a/mysql-test/main/sp.result +++ b/mysql-test/main/sp.result @@ -8972,3 +8972,15 @@ select @counter; 5 drop function f1; drop table t1,t2; +# +# MDEV-31877: ASAN errors in Exec_time_tracker::get_cycles with innodb slow log verbosity +# +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2); +SET @tmp=@@log_slow_verbosity; +SET SESSION log_slow_verbosity= 'innodb'; +BEGIN NOT ATOMIC DECLARE r ROW TYPE OF t1 DEFAULT (SELECT * FROM t1); SELECT r.a; END $ +r.a +1 +SET SESSION log_slow_verbosity= @tmp; +DROP TABLE t1; diff --git a/mysql-test/main/sp.test b/mysql-test/main/sp.test index e8da0bd2209..6950fbddd65 100644 --- a/mysql-test/main/sp.test +++ b/mysql-test/main/sp.test @@ -10573,3 +10573,19 @@ select f1(col1) from t2 order by col2 desc limit 5; select @counter; drop function f1; drop table t1,t2; + +--echo # +--echo # MDEV-31877: ASAN errors in Exec_time_tracker::get_cycles with innodb slow log verbosity +--echo # +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2); +SET @tmp=@@log_slow_verbosity; +SET SESSION log_slow_verbosity= 'innodb'; +--delimiter $ +BEGIN NOT ATOMIC DECLARE r ROW TYPE OF t1 DEFAULT (SELECT * FROM t1); SELECT r.a; END $ +--delimiter ; + +SET SESSION log_slow_verbosity= @tmp; +# Cleanup +DROP TABLE t1; + diff --git a/sql/sp_head.cc b/sql/sp_head.cc index ecb3fa2d6cd..f727356b5f6 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -3807,7 +3807,6 @@ int sp_instr_set::exec_core(THD *thd, uint *nextp) { int res= get_rcontext(thd)->set_variable(thd, m_offset, &m_value); - delete_explain_query(thd->lex); *nextp = m_ip+1; return res; } @@ -3849,7 +3848,6 @@ sp_instr_set_row_field::exec_core(THD *thd, uint *nextp) int res= get_rcontext(thd)->set_variable_row_field(thd, m_offset, m_field_offset, &m_value); - delete_explain_query(thd->lex); *nextp= m_ip + 1; return res; } @@ -3897,7 +3895,6 @@ sp_instr_set_row_field_by_name::exec_core(THD *thd, uint *nextp) int res= get_rcontext(thd)->set_variable_row_field_by_name(thd, m_offset, m_field_name, &m_value); - delete_explain_query(thd->lex); *nextp= m_ip + 1; return res; } From 2aea9387497cecb5668ef605b8f80886f9de812c Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 10 Aug 2023 16:13:32 +0300 Subject: [PATCH 2/2] MDEV-31893 Valgrind reports issues in main.join_cache_notasan This is also related to MDEV-31348 Assertion `last_key_entry >= end_pos' failed in virtual bool JOIN_CACHE_HASHED::put_record() Valgrind exposed a problem with the join_cache for hash joins: =25636== Conditional jump or move depends on uninitialised value(s) ==25636== at 0xA8FF4E: JOIN_CACHE_HASHED::init_hash_table() (sql_join_cache.cc:2901) The reason for this was that avg_record_length contained a random value if one had used SET optimizer_switch='optimize_join_buffer_size=off'. This causes either 'random size' memory to be allocated (up to join_buffer_size) which can increase memory usage or, if avg_record_length is less than the row size, memory overwrites in thd->mem_root, which is bad. Fixed by setting avg_record_length in JOIN_CACHE_HASHED::init() before it's used. There is no test case for MDEV-31893 as valgrind of join_cache_notasan checks that. I added a test case for MDEV-31348. --- include/no_valgrind_without_big.inc | 13 +++++++++++ mysql-test/main/join_cache.result | 16 ++++++++++++++ mysql-test/main/join_cache.test | 19 ++++++++++++++++ mysql-test/main/join_cache_notasan.test | 3 +++ sql/sql_join_cache.cc | 29 +++++++++++++++---------- sql/sql_join_cache.h | 1 + 6 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 include/no_valgrind_without_big.inc diff --git a/include/no_valgrind_without_big.inc b/include/no_valgrind_without_big.inc new file mode 100644 index 00000000000..ea1f2ac91ab --- /dev/null +++ b/include/no_valgrind_without_big.inc @@ -0,0 +1,13 @@ +# include/no_valgrind_without_big.inc +# +# If we are running with Valgrind ($VALGRIND_TEST <> 0) than the resource +# consumption (storage space needed, runtime ...) will be extreme. +# Therefore we require that the option "--big-test" is also set. +# + +if ($VALGRIND_TEST) { + if (!$BIG_TEST) + { + --skip Need "--big-test" when running with Valgrind + } +} diff --git a/mysql-test/main/join_cache.result b/mysql-test/main/join_cache.result index 20980d09001..92c04bb002b 100644 --- a/mysql-test/main/join_cache.result +++ b/mysql-test/main/join_cache.result @@ -6256,3 +6256,19 @@ length(concat(t1.f,t2.f)) DROP TABLE t; set @@optimizer_switch=@org_optimizer_switch; set @@join_buffer_size=@org_join_buffer_size; +# +# MDEV-31348 Assertion `last_key_entry >= end_pos' failed in +# virtual bool JOIN_CACHE_HASHED::put_record() +# +SET JOIN_buffer_size=1; +Warnings: +Warning 1292 Truncated incorrect join_buffer_size value: '1' +SET SESSION JOIN_cache_level=4; +SET SESSION optimizer_switch='optimize_JOIN_buffer_size=OFF'; +SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name); +ERROR HY001: Could not create a join buffer. Please check and adjust the value of the variables 'JOIN_BUFFER_SIZE (128)' and 'JOIN_BUFFER_SPACE_LIMIT (2097152)' +SET JOIN_buffer_size=16384; +SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name); +# +# End of 10.4 tests +# diff --git a/mysql-test/main/join_cache.test b/mysql-test/main/join_cache.test index 43ce3b97ad1..f8723447efe 100644 --- a/mysql-test/main/join_cache.test +++ b/mysql-test/main/join_cache.test @@ -4231,3 +4231,22 @@ SELECT length(concat(t1.f,t2.f)) FROM t t1, t t2; DROP TABLE t; set @@optimizer_switch=@org_optimizer_switch; set @@join_buffer_size=@org_join_buffer_size; + +--echo # +--echo # MDEV-31348 Assertion `last_key_entry >= end_pos' failed in +--echo # virtual bool JOIN_CACHE_HASHED::put_record() +--echo # + +SET JOIN_buffer_size=1; +SET SESSION JOIN_cache_level=4; +SET SESSION optimizer_switch='optimize_JOIN_buffer_size=OFF'; +--error ER_OUTOFMEMORY +SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name); +SET JOIN_buffer_size=16384; +--disable_result_log +SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name); +--enable_result_log + +--echo # +--echo # End of 10.4 tests +--echo # diff --git a/mysql-test/main/join_cache_notasan.test b/mysql-test/main/join_cache_notasan.test index cfdfe4eff18..406303ef7b3 100644 --- a/mysql-test/main/join_cache_notasan.test +++ b/mysql-test/main/join_cache_notasan.test @@ -2,7 +2,10 @@ # Tests that should be in join_cache but cannot be run with ASAN --source include/have_64bit.inc +# Disable asan it asan builds crashes when trying to allocate too much memory --source include/not_asan.inc +# Valgrind is useful here, but very slow as lots of memory is allocated +--source include/no_valgrind_without_big.inc --source include/have_innodb.inc --echo # diff --git a/sql/sql_join_cache.cc b/sql/sql_join_cache.cc index 1319fd59a99..f1dd23d9618 100644 --- a/sql/sql_join_cache.cc +++ b/sql/sql_join_cache.cc @@ -800,6 +800,18 @@ size_t JOIN_CACHE::get_min_join_buffer_size() } +size_t JOIN_CACHE::calc_avg_record_length() +{ + size_t len= 0; + for (JOIN_TAB *tab= start_tab; tab != join_tab; + tab= next_linear_tab(join, tab, WITHOUT_BUSH_ROOTS)) + { + len+= tab->get_used_fieldlength(); + } + len+= get_record_max_affix_length(); + return len; +} + /* Get the maximum possible size of the cache join buffer @@ -822,9 +834,9 @@ size_t JOIN_CACHE::get_min_join_buffer_size() 'max_buff_size' in order to use it directly at the next invocations of the function. - RETURN VALUE - The maximum possible size of the join buffer of this cache + The maximum possible size of the join buffer of this cache + avg_record_length is also updated if optimize_buff_size != 0 */ size_t JOIN_CACHE::get_max_join_buffer_size(bool optimize_buff_size, @@ -839,19 +851,13 @@ size_t JOIN_CACHE::get_max_join_buffer_size(bool optimize_buff_size, return max_buff_size= limit_sz; size_t max_sz; - size_t len= 0; + size_t len; double max_records, partial_join_cardinality= (join_tab-1)->get_partial_join_cardinality(); /* Expected join buffer space used for one record */ size_t space_per_record; - for (JOIN_TAB *tab= start_tab; tab != join_tab; - tab= next_linear_tab(join, tab, WITHOUT_BUSH_ROOTS)) - { - len+= tab->get_used_fieldlength(); - } - len+= get_record_max_affix_length(); - avg_record_length= len; + len= avg_record_length= calc_avg_record_length(); len+= get_max_key_addon_space_per_record() + avg_aux_buffer_incr; space_per_record= len; @@ -2786,7 +2792,6 @@ bool JOIN_CACHE_BKAH::save_explain_data(EXPLAIN_BKA_TYPE *explain) int JOIN_CACHE_HASHED::init(bool for_explain) { TABLE_REF *ref= &join_tab->ref; - DBUG_ENTER("JOIN_CACHE_HASHED::init"); hash_table= 0; @@ -2873,6 +2878,8 @@ int JOIN_CACHE_HASHED::init_hash_table() hash_table= 0; key_entries= 0; + avg_record_length= calc_avg_record_length(); + /* Calculate the minimal possible value of size_of_key_ofs greater than 1 */ uint max_size_of_key_ofs= MY_MAX(2, get_size_of_rec_offset()); for (size_of_key_ofs= 2; diff --git a/sql/sql_join_cache.h b/sql/sql_join_cache.h index 8bdce1bd592..b0cfb674ef9 100644 --- a/sql/sql_join_cache.h +++ b/sql/sql_join_cache.h @@ -130,6 +130,7 @@ protected: case 4: int4store(ptr, (uint32) ofs); return; } } + size_t calc_avg_record_length(); /* The maximum total length of the fields stored for a record in the cache.