From 2b8b7394a129ab27225a1284bab253a6714aaf03 Mon Sep 17 00:00:00 2001 From: mkaruza Date: Sun, 28 Jun 2020 23:03:38 +0200 Subject: [PATCH 1/9] MDEV-22222: Assertion `state() == s_executing || state() == s_preparing || state() == s_prepared || state() == s_must_abort || state() == s_aborting || state() == s_cert_failed || state() == s_must_replay' failed in wsrep::transaction::before_rollback LOCK TABLE will do implicit commit, we need to properly handle transaction after commit. --- .../galera_lock_tables_in_transaction.result | 12 +++++++++++ .../t/galera_lock_tables_in_transaction.test | 21 +++++++++++++++++++ sql/sql_parse.cc | 6 ++++++ 3 files changed, 39 insertions(+) create mode 100644 mysql-test/suite/galera/r/galera_lock_tables_in_transaction.result create mode 100644 mysql-test/suite/galera/t/galera_lock_tables_in_transaction.test diff --git a/mysql-test/suite/galera/r/galera_lock_tables_in_transaction.result b/mysql-test/suite/galera/r/galera_lock_tables_in_transaction.result new file mode 100644 index 00000000000..68691a4efd2 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_lock_tables_in_transaction.result @@ -0,0 +1,12 @@ +connection node_2; +connection node_1; +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +START TRANSACTION; +INSERT INTO t1 VALUES (1); +LOCK TABLES t2 READ; +ERROR 42S02: Table 'test.t2' doesn't exist +START TRANSACTION; +INSERT INTO t1 VALUES (1); +LOCK TABLES t1 READ; +UNLOCK TABLES; +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_lock_tables_in_transaction.test b/mysql-test/suite/galera/t/galera_lock_tables_in_transaction.test new file mode 100644 index 00000000000..5cb7347639c --- /dev/null +++ b/mysql-test/suite/galera/t/galera_lock_tables_in_transaction.test @@ -0,0 +1,21 @@ +# +# Check `LOCK TABLES` command with or without existing table in database. +# Test case for MDEV-22222 / MDEV-22223 +# + +--source include/galera_cluster.inc +--source include/have_innodb.inc + +CREATE TABLE t1 (a INT) ENGINE=InnoDB; + +START TRANSACTION; +INSERT INTO t1 VALUES (1); +--error ER_NO_SUCH_TABLE +LOCK TABLES t2 READ; + +START TRANSACTION; +INSERT INTO t1 VALUES (1); +LOCK TABLES t1 READ; +UNLOCK TABLES; + +DROP TABLE t1; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index a7f2f472b0a..69ffbed1ca6 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4998,6 +4998,12 @@ mysql_execute_command(THD *thd) if (res) goto error; +#ifdef WITH_WSREP + /* Clean up the previous transaction on implicit commit. */ + if (wsrep_on(thd) && !wsrep_not_committed(thd) && wsrep_after_statement(thd)) + goto error; +#endif + /* We can't have any kind of table locks while backup is active */ if (thd->current_backup_stage != BACKUP_FINISHED) { From 3f2044ae99633ce6d9c756bb6b045efc0707b4b5 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 26 Jun 2020 13:42:04 +0300 Subject: [PATCH 2/9] MDEV-22535 TABLE::initialize_quick_structures() takes 0.5% in oltp_read_only - Removed not needed bzero in void TABLE::initialize_quick_structures(). - Replaced bzero with TRASH_ALLOC() to have this change verfied with memory checkers - Added missing table->quick_keys.is_set in table_cond_selectivity() --- sql/sql_select.cc | 2 +- sql/table.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index e7f593326a4..1738cf5d18f 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -8573,7 +8573,7 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, /* Check if we have a prefix of key=const that matches a quick select. */ - if (!is_hash_join_key_no(key)) + if (!is_hash_join_key_no(key) && table->quick_keys.is_set(key)) { key_part_map quick_key_map= (key_part_map(1) << table->quick_key_parts[key]) - 1; if (table->quick_rows[key] && diff --git a/sql/table.cc b/sql/table.cc index 1877c3b7c76..c0cfc5ca3db 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -9358,10 +9358,10 @@ bool TABLE::export_structure(THD *thd, Row_definition_list *defs) void TABLE::initialize_quick_structures() { - bzero(quick_rows, sizeof(quick_rows)); - bzero(quick_key_parts, sizeof(quick_key_parts)); - bzero(quick_costs, sizeof(quick_costs)); - bzero(quick_n_ranges, sizeof(quick_n_ranges)); + TRASH_ALLOC(quick_rows, sizeof(quick_rows)); + TRASH_ALLOC(quick_key_parts, sizeof(quick_key_parts)); + TRASH_ALLOC(quick_costs, sizeof(quick_costs)); + TRASH_ALLOC(quick_n_ranges, sizeof(quick_n_ranges)); } /* From 29f9e679adc90adf5d3c6e08da947789c9c2ac8b Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 14 Aug 2019 23:46:47 +0300 Subject: [PATCH 3/9] Don't copy uninitialized bytes when copying varstrings When using field_conv(), which is called in case of field1=field2 copy in fill_records(), full varstring's was copied, including unitialized bytes. This caused valgrind to compilain about usage of unitialized bytes when using Aria static length records. Fixed by not using memcpy when copying varstrings but instead just copy the real bytes. --- sql/field.cc | 11 +++++++++++ sql/field.h | 7 +------ sql/handler.h | 3 ++- storage/maria/ha_maria.cc | 2 +- storage/myisam/ha_myisam.cc | 5 +++-- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/sql/field.cc b/sql/field.cc index 155d8a4eb0d..1e2a711c956 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7613,6 +7613,17 @@ int Field_varstring::save_field_metadata(uchar *metadata_ptr) return 2; } + +bool Field_varstring::memcpy_field_possible(const Field *from) const +{ + return (Field_str::memcpy_field_possible(from) && + !compression_method() == !from->compression_method() && + length_bytes == ((Field_varstring*) from)->length_bytes && + (table->file && !(table->file->ha_table_flags() & + HA_RECORD_MUST_BE_CLEAN_ON_WRITE))); +} + + int Field_varstring::store(const char *from,size_t length,CHARSET_INFO *cs) { ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED; diff --git a/sql/field.h b/sql/field.h index db78efe0754..34d6684571b 100644 --- a/sql/field.h +++ b/sql/field.h @@ -3463,12 +3463,7 @@ public: length_bytes : 0); } Copy_func *get_copy_func(const Field *from) const; - bool memcpy_field_possible(const Field *from) const - { - return Field_str::memcpy_field_possible(from) && - !compression_method() == !from->compression_method() && - length_bytes == ((Field_varstring*) from)->length_bytes; - } + bool memcpy_field_possible(const Field *from) const; int store(const char *to,size_t length,CHARSET_INFO *charset); using Field_str::store; #ifdef HAVE_valgrind_or_MSAN diff --git a/sql/handler.h b/sql/handler.h index af9fa068891..16fcab1e755 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -200,7 +200,8 @@ enum enum_alter_inplace_result { #define HA_HAS_NEW_CHECKSUM (1ULL << 38) #define HA_CAN_VIRTUAL_COLUMNS (1ULL << 39) #define HA_MRR_CANT_SORT (1ULL << 40) -#define HA_RECORD_MUST_BE_CLEAN_ON_WRITE (1ULL << 41) /* unused */ +/* All of VARCHAR is stored, including bytes after real varchar data */ +#define HA_RECORD_MUST_BE_CLEAN_ON_WRITE (1ULL << 41) /* This storage engine supports condition pushdown diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index ec2a88de4df..c0163473f3a 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -1162,7 +1162,7 @@ int ha_maria::open(const char *name, int mode, uint test_if_locked) that all bytes in the row is properly reset. */ if (file->s->data_file_type == STATIC_RECORD && - (file->s->has_varchar_fields | file->s->has_null_fields)) + (file->s->has_varchar_fields || file->s->has_null_fields)) int_table_flags|= HA_RECORD_MUST_BE_CLEAN_ON_WRITE; for (i= 0; i < table->s->keys; i++) diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index fd2e8fd9425..0d9a2c6c18e 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -863,8 +863,9 @@ int ha_myisam::open(const char *name, int mode, uint test_if_locked) the full row to ensure we don't get any errors from valgrind and that all bytes in the row is properly reset. */ - if ((file->s->options & HA_OPTION_PACK_RECORD) && - (file->s->has_varchar_fields | file->s->has_null_fields)) + if (!(file->s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) && + (file->s->has_varchar_fields || file->s->has_null_fields)) int_table_flags|= HA_RECORD_MUST_BE_CLEAN_ON_WRITE; for (i= 0; i < table->s->keys; i++) From 65f831d17c84900c1faea49164688e2f5ce59563 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 28 Jun 2020 20:07:32 +0300 Subject: [PATCH 4/9] Fixed bugs found by valgrind - Some of the bug fixes are backports from 10.5! - The fix in innobase/fil/fil0fil.cc is just a backport to get less error messages in mysqld.1.err when running with valgrind. - Renamed HAVE_valgrind_or_MSAN to HAVE_valgrind --- client/mysqltest.cc | 11 ++++--- include/my_sys.h | 2 ++ include/my_valgrind.h | 40 +++++++++++------------ mysql-test/main/sp-big.result | 4 +-- mysql-test/main/sp-big.test | 13 +++++++- sql/field.cc | 2 +- sql/field.h | 7 ++-- sql/ha_partition.cc | 2 +- sql/item_cmpfunc.cc | 2 ++ sql/item_subselect.cc | 2 +- sql/opt_range.cc | 1 + sql/session_tracker.cc | 5 +-- sql/sql_select.cc | 44 ++++++++++++++++++-------- sql/table.cc | 8 ++--- sql/unireg.cc | 4 +-- storage/innobase/btr/btr0cur.cc | 8 ++--- storage/innobase/buf/buf0buddy.cc | 4 +-- storage/innobase/buf/buf0lru.cc | 12 +++---- storage/innobase/data/data0data.cc | 4 +-- storage/innobase/fil/fil0fil.cc | 3 +- storage/innobase/handler/ha_innodb.cc | 8 ++--- storage/innobase/include/dict0stats.ic | 4 +-- storage/innobase/include/srv0mon.h | 6 ++-- storage/innobase/page/page0cur.cc | 8 ++--- storage/innobase/row/row0ftsort.cc | 4 +-- storage/innobase/row/row0ins.cc | 4 +-- storage/innobase/row/row0log.cc | 32 +++++++++---------- storage/innobase/row/row0merge.cc | 22 ++++++------- storage/innobase/row/row0sel.cc | 20 ++++++------ storage/innobase/row/row0upd.cc | 4 +-- storage/innobase/trx/trx0trx.cc | 2 +- storage/tokudb/ha_tokudb.cc | 5 +++ 32 files changed, 171 insertions(+), 126 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 1fed08ea0af..316e52d95b4 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -10179,7 +10179,7 @@ void append_replace_regex(char* expr, char *expr_end, struct st_replace_regex* r /* Allow variable for the *entire* list of replacements */ if (*p == '$') { - const char *v_end; + const char *v_end= 0; VAR *val= var_get(p, &v_end, 0, 1); if (val) @@ -10820,7 +10820,7 @@ REPLACE *init_replace(char * *from, char * *to,uint count, for (i=1 ; i <= found_sets ; i++) { pos=from[found_set[i-1].table_offset]; - rep_str[i].found= !memcmp(pos, "\\^", 3) ? 2 : 1; + rep_str[i].found= !strncmp(pos, "\\^", 3) ? 2 : 1; rep_str[i].replace_string=to_array[found_set[i-1].table_offset]; rep_str[i].to_offset=found_set[i-1].found_offset-start_at_word(pos); rep_str[i].from_offset=found_set[i-1].found_offset-replace_len(pos)+ @@ -11132,13 +11132,16 @@ void replace_dynstr_append_mem(DYNAMIC_STRING *ds, const char *val, size_t len) { /* Convert to lower case, and do this first */ char *c= lower; - for (const char *v= val; *v; v++) + for (const char *v= val, *end_v= v + len; v < end_v; v++) *c++= my_tolower(charset_info, *v); *c= '\0'; /* Copy from this buffer instead */ } else - memcpy(lower, val, len+1); + { + memcpy(lower, val, len); + lower[len]= 0; + } fix_win_paths(lower, len); val= lower; } diff --git a/include/my_sys.h b/include/my_sys.h index 61551d38337..60f5613eec1 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -535,6 +535,7 @@ static inline int my_b_read(IO_CACHE *info, uchar *Buffer, size_t Count) static inline int my_b_write(IO_CACHE *info, const uchar *Buffer, size_t Count) { + MEM_CHECK_DEFINED(Buffer, Count); if (info->write_pos + Count <= info->write_end) { memcpy(info->write_pos, Buffer, Count); @@ -556,6 +557,7 @@ static inline int my_b_get(IO_CACHE *info) static inline my_bool my_b_write_byte(IO_CACHE *info, uchar chr) { + MEM_CHECK_DEFINED(&chr, 1); if (info->write_pos >= info->write_end) if (my_b_flush_io_cache(info, 1)) return 1; diff --git a/include/my_valgrind.h b/include/my_valgrind.h index 1de6714700b..c181a8c6b35 100644 --- a/include/my_valgrind.h +++ b/include/my_valgrind.h @@ -24,15 +24,19 @@ # define __SANITIZE_ADDRESS__ 1 #endif -#ifdef HAVE_valgrind -#define IF_VALGRIND(A,B) A -#else -#define IF_VALGRIND(A,B) B -#endif - -#if defined(HAVE_VALGRIND_MEMCHECK_H) && defined(HAVE_valgrind) +#if __has_feature(memory_sanitizer) +# include +# define HAVE_valgrind +# define MEM_UNDEFINED(a,len) __msan_allocated_memory(a,len) +# define MEM_MAKE_DEFINED(a,len) __msan_unpoison(a,len) +# define MEM_NOACCESS(a,len) ((void) 0) +# define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0) +# define MEM_CHECK_DEFINED(a,len) __msan_check_mem_is_initialized(a,len) +# define MEM_GET_VBITS(a,b,len) __msan_copy_shadow(b,a,len) +# define MEM_SET_VBITS(a,b,len) __msan_copy_shadow(a,b,len) +# define REDZONE_SIZE 8 +#elif defined(HAVE_VALGRIND_MEMCHECK_H) && defined(HAVE_valgrind) # include -# define HAVE_valgrind_or_MSAN # define MEM_UNDEFINED(a,len) VALGRIND_MAKE_MEM_UNDEFINED(a,len) # define MEM_MAKE_DEFINED(a,len) VALGRIND_MAKE_MEM_DEFINED(a,len) # define MEM_NOACCESS(a,len) VALGRIND_MAKE_MEM_NOACCESS(a,len) @@ -53,17 +57,6 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ # define MEM_GET_VBITS(a,b,len) ((void) 0) # define MEM_SET_VBITS(a,b,len) ((void) 0) # define REDZONE_SIZE 8 -#elif __has_feature(memory_sanitizer) -# include -# define HAVE_valgrind_or_MSAN -# define MEM_UNDEFINED(a,len) __msan_allocated_memory(a,len) -# define MEM_MAKE_DEFINED(a,len) __msan_unpoison(a,len) -# define MEM_NOACCESS(a,len) ((void) 0) -# define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0) -# define MEM_CHECK_DEFINED(a,len) __msan_check_mem_is_initialized(a,len) -# define MEM_GET_VBITS(a,b,len) __msan_copy_shadow(b,a,len) -# define MEM_SET_VBITS(a,b,len) __msan_copy_shadow(a,b,len) -# define REDZONE_SIZE 8 #else # define MEM_UNDEFINED(a,len) ((void) (a), (void) (len)) # define MEM_MAKE_DEFINED(a,len) ((void) 0) @@ -73,7 +66,14 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ # define MEM_GET_VBITS(a,b,len) ((void) 0) # define MEM_SET_VBITS(a,b,len) ((void) 0) # define REDZONE_SIZE 0 -#endif /* HAVE_VALGRIND_MEMCHECK_H */ +#endif /* __has_feature(memory_sanitizer) */ + +#ifdef HAVE_valgrind +#define IF_VALGRIND(A,B) A +#else +#define IF_VALGRIND(A,B) B +#endif + #ifdef TRASH_FREED_MEMORY /* diff --git a/mysql-test/main/sp-big.result b/mysql-test/main/sp-big.result index e12136eb36d..611ac9b74e9 100644 --- a/mysql-test/main/sp-big.result +++ b/mysql-test/main/sp-big.result @@ -80,8 +80,8 @@ end// insert t1 select seq, seq, 1, 1, seq, seq, seq from seq_1_to_2000; set @before=unix_timestamp(); call select_test(); -select unix_timestamp() - @before < 60; -unix_timestamp() - @before < 60 +select unix_timestamp() - @before < @time; +unix_timestamp() - @before < @time 1 drop procedure select_test; drop table t1; diff --git a/mysql-test/main/sp-big.test b/mysql-test/main/sp-big.test index 4220541697e..043e737105a 100644 --- a/mysql-test/main/sp-big.test +++ b/mysql-test/main/sp-big.test @@ -112,6 +112,17 @@ delimiter ;// insert t1 select seq, seq, 1, 1, seq, seq, seq from seq_1_to_2000; set @before=unix_timestamp(); call select_test(); -select unix_timestamp() - @before < 60; + +--let $time=60 +if ($VALGRIND_TEST) +{ + --let $time=600 +} + +--disable_query_log +--eval set @time=$time; +--enable_query_log + +select unix_timestamp() - @before < @time; drop procedure select_test; drop table t1; diff --git a/sql/field.cc b/sql/field.cc index 1e2a711c956..8accfb35b0b 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7686,7 +7686,7 @@ my_decimal *Field_varstring::val_decimal(my_decimal *decimal_value) } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind void Field_varstring::mark_unused_memory_as_defined() { uint used_length= get_length(); diff --git a/sql/field.h b/sql/field.h index 34d6684571b..83cdb903b20 100644 --- a/sql/field.h +++ b/sql/field.h @@ -826,7 +826,7 @@ public: return store(ls.str, (uint) ls.length, cs); } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind /** Mark unused memory in the field as defined. Mainly used to ensure that if we write full field to disk (for example in @@ -3466,7 +3466,7 @@ public: bool memcpy_field_possible(const Field *from) const; int store(const char *to,size_t length,CHARSET_INFO *charset); using Field_str::store; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind void mark_unused_memory_as_defined(); #endif double val_real(void); @@ -4395,7 +4395,8 @@ public: :Type_handler_hybrid_field_type(&type_handler_null), compression_method_ptr(0), comment(null_clex_str), - on_update(NULL), length(0), invisible(VISIBLE), decimals(0), + on_update(NULL), length(0), invisible(VISIBLE), char_length(0), + decimals(0), flags(0), pack_length(0), key_length(0), unireg_check(Field::NONE), interval(0), charset(&my_charset_bin), srid(0), geom_type(Field::GEOM_GEOMETRY), diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 8165474ccea..e19f21de006 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -2548,7 +2548,7 @@ register_query_cache_dependant_tables(THD *thd, sub_elem= subpart_it++; part= i * num_subparts + j; /* we store the end \0 as part of the key */ - end= strmov(engine_pos, sub_elem->partition_name); + end= strmov(engine_pos, sub_elem->partition_name) + 1; length= (uint)(end - engine_key); /* Copy the suffix also to query cache key */ memcpy(query_cache_key_end, engine_key_end, (end - engine_key_end)); diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index ee371b8f896..a0a3631b15b 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -1176,6 +1176,8 @@ longlong Item_func_truth::val_int() bool Item_in_optimizer::is_top_level_item() { + if (invisible_mode()) + return FALSE; return ((Item_in_subselect *)args[1])->is_top_level_item(); } diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index ce2427bce91..c0a98aab183 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -717,7 +717,7 @@ bool Item_subselect::exec() push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, ER_UNKNOWN_ERROR, "DBUG: Item_subselect::exec %.*s", - print.length(),print.c_ptr()); + print.length(),print.ptr()); ); /* Do not execute subselect in case of a fatal error diff --git a/sql/opt_range.cc b/sql/opt_range.cc index a936184de0d..0edc1665ac9 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -2457,6 +2457,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, param.imerge_cost_buff_size= 0; param.using_real_indexes= TRUE; param.remove_jump_scans= TRUE; + param.is_ror_scan= 0; param.remove_false_where_parts= remove_false_parts_of_where; param.force_default_mrr= ordered_output; param.possible_keys.clear_all(); diff --git a/sql/session_tracker.cc b/sql/session_tracker.cc index 65d600b9b5a..7133e45ab11 100644 --- a/sql/session_tracker.cc +++ b/sql/session_tracker.cc @@ -380,9 +380,10 @@ bool Session_sysvars_tracker::enable(THD *thd) bool Session_sysvars_tracker::update(THD *thd, set_var *var) { vars_list tool_list; + size_t length= 1; void *copy= var->save_result.string_value.str ? my_memdup(var->save_result.string_value.str, - var->save_result.string_value.length + 1, + length= var->save_result.string_value.length + 1, MYF(MY_WME | MY_THREAD_SPECIFIC)) : my_strdup("", MYF(MY_WME | MY_THREAD_SPECIFIC)); @@ -402,7 +403,7 @@ bool Session_sysvars_tracker::update(THD *thd, set_var *var) m_parsed= true; orig_list.copy(&tool_list, thd); orig_list.construct_var_list(thd->variables.session_track_system_variables, - var->save_result.string_value.length + 1); + length); return false; } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1738cf5d18f..88a1cdedeac 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -6176,9 +6176,11 @@ add_ft_keys(DYNAMIC_ARRAY *keyuse_array, keyuse.keypart= FT_KEYPART; keyuse.used_tables=cond_func->key_item()->used_tables(); keyuse.optimize= 0; + keyuse.ref_table_rows= 0; keyuse.keypart_map= 0; keyuse.sj_pred_no= UINT_MAX; keyuse.validity_ref= 0; + keyuse.null_rejecting= FALSE; return insert_dynamic(keyuse_array,(uchar*) &keyuse); } @@ -17966,25 +17968,31 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, inherit the default value that is defined for the field referred by the Item_field object from which 'field' has been created. */ - const Field *orig_field= default_field[i]; + Field *orig_field= default_field[i]; /* Get the value from default_values */ if (orig_field->is_null_in_record(orig_field->table->s->default_values)) field->set_null(); else { + /* + Copy default value. We have to use field_conv() for copy, instead of + memcpy(), because bit_fields may be stored differently + */ + my_ptrdiff_t ptr_diff= (orig_field->table->s->default_values - + orig_field->table->record[0]); field->set_notnull(); - memcpy(field->ptr, - orig_field->ptr_in_record(orig_field->table->s->default_values), - field->pack_length_in_rec()); + orig_field->move_field_offset(ptr_diff); + field_conv(field, orig_field); + orig_field->move_field_offset(-ptr_diff); } - } + } if (from_field[i]) { /* Not a table Item */ copy->set(field,from_field[i],save_sum_fields); copy++; } - length=field->pack_length(); + length=field->pack_length_in_rec(); pos+= length; /* Make entry for create table */ @@ -18006,6 +18014,9 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, // fix table name in field entry field->set_table_name(&table->alias); } + /* Handle group_null_items */ + bzero(pos, table->s->reclength - (pos - table->record[0])); + MEM_CHECK_DEFINED(table->record[0], table->s->reclength); param->copy_field_end=copy; param->recinfo= recinfo; // Pointer to after last field @@ -18275,8 +18286,9 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, goto err; } - // Make empty record so random data is not written to disk - empty_record(table); + /* record[0] and share->default_values should now have been set up */ + MEM_CHECK_DEFINED(table->record[0], table->s->reclength); + MEM_CHECK_DEFINED(share->default_values, table->s->reclength); thd->mem_root= mem_root_save; @@ -18571,7 +18583,11 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, (*recinfo)->type= FIELD_CHECK; (*recinfo)->length= MARIA_UNIQUE_HASH_LENGTH; (*recinfo)++; - share->reclength+= MARIA_UNIQUE_HASH_LENGTH; + + /* Avoid warnings from valgrind */ + bzero(table->record[0]+ share->reclength, MARIA_UNIQUE_HASH_LENGTH); + bzero(share->default_values+ share->reclength, MARIA_UNIQUE_HASH_LENGTH); + share->reclength+= MARIA_UNIQUE_HASH_LENGTH; } else { @@ -18765,7 +18781,10 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, (*recinfo)->type= FIELD_CHECK; (*recinfo)->length=MI_UNIQUE_HASH_LENGTH; (*recinfo)++; - share->reclength+=MI_UNIQUE_HASH_LENGTH; + /* Avoid warnings from valgrind */ + bzero(table->record[0]+ share->reclength, MI_UNIQUE_HASH_LENGTH); + bzero(share->default_values+ share->reclength, MI_UNIQUE_HASH_LENGTH); + share->reclength+= MI_UNIQUE_HASH_LENGTH; } else { @@ -19357,11 +19376,11 @@ bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, If it is not heap (in-memory) table then convert index to unique constrain. */ + MEM_CHECK_DEFINED(table->record[0], table->s->reclength); if (create_internal_tmp_table(table, keyinfo, start_recinfo, recinfo, options)) return TRUE; - // Make empty record so random data is not written to disk - empty_record(table); + MEM_CHECK_DEFINED(table->record[0], table->s->reclength); } if (open_tmp_table(table)) return TRUE; @@ -27698,7 +27717,6 @@ AGGR_OP::prepare_tmp_table() join->select_options)) return true; (void) table->file->extra(HA_EXTRA_WRITE_CACHE); - empty_record(table); } /* If it wasn't already, start index scan for grouping using table index. */ if (!table->file->inited && table->group && diff --git a/sql/table.cc b/sql/table.cc index c0cfc5ca3db..ec2d86e232e 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -1644,8 +1644,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, share->rec_buff_length= rec_buff_length; if (!(record= (uchar *) alloc_root(&share->mem_root, rec_buff_length))) goto err; /* purecov: inspected */ - MEM_NOACCESS(record, rec_buff_length); - MEM_UNDEFINED(record, share->reclength); + MEM_NOACCESS(record + share->reclength, rec_buff_length - share->reclength); share->default_values= record; memcpy(record, frm_image + record_offset, share->reclength); @@ -3273,7 +3272,6 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, if (!(record= (uchar*) alloc_root(&outparam->mem_root, share->rec_buff_length * records))) goto err; /* purecov: inspected */ - MEM_NOACCESS(record, share->rec_buff_length * records); } for (i= 0; i < 3;) @@ -3282,8 +3280,10 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, if (++i < records) record+= share->rec_buff_length; } + /* Mark bytes between records as not accessable to catch overrun bugs */ for (i= 0; i < records; i++) - MEM_UNDEFINED(outparam->record[i], share->reclength); + MEM_NOACCESS(outparam->record[i] + share->reclength, + share->rec_buff_length - share->reclength); if (!(field_ptr = (Field **) alloc_root(&outparam->mem_root, (uint) ((share->fields+1)* diff --git a/sql/unireg.cc b/sql/unireg.cc index b7eb1cd3457..cccb09a9fb9 100644 --- a/sql/unireg.cc +++ b/sql/unireg.cc @@ -436,8 +436,8 @@ LEX_CUSTRING build_frm_image(THD *thd, const LEX_CSTRING *table, pos+= create_info->comment.length; } - memcpy(frm_ptr + filepos, forminfo, 288); - pos= frm_ptr + filepos + 288; + memcpy(frm_ptr + filepos, forminfo, FRM_FORMINFO_SIZE); + pos= frm_ptr + filepos + FRM_FORMINFO_SIZE; if (pack_fields(&pos, create_fields, create_info, data_offset)) goto err; diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index e7650b20507..d9684528195 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1179,12 +1179,12 @@ btr_cur_search_to_nth_level_func( ut_ad(!(index->type & DICT_FTS)); ut_ad(index->page != FIL_NULL); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(&cursor->up_match, sizeof cursor->up_match); MEM_UNDEFINED(&cursor->up_bytes, sizeof cursor->up_bytes); MEM_UNDEFINED(&cursor->low_match, sizeof cursor->low_match); MEM_UNDEFINED(&cursor->low_bytes, sizeof cursor->low_bytes); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ #ifdef UNIV_DEBUG cursor->up_match = ULINT_UNDEFINED; cursor->low_match = ULINT_UNDEFINED; @@ -3295,12 +3295,12 @@ btr_cur_optimistic_insert( const page_size_t& page_size = block->page.size; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind if (page_size.is_compressed()) { MEM_CHECK_DEFINED(page, page_size.logical()); MEM_CHECK_DEFINED(block->page.zip.data, page_size.physical()); } -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ leaf = page_is_leaf(page); diff --git a/storage/innobase/buf/buf0buddy.cc b/storage/innobase/buf/buf0buddy.cc index ed36bcb9703..f1849dc28bf 100644 --- a/storage/innobase/buf/buf0buddy.cc +++ b/storage/innobase/buf/buf0buddy.cc @@ -393,9 +393,9 @@ buf_buddy_block_free( HASH_DELETE(buf_page_t, hash, buf_pool->zip_hash, fold, bpage); ut_d(memset(buf, 0, srv_page_size)); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(buf, srv_page_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ block = (buf_block_t*) bpage; buf_page_mutex_enter(block); diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index c81975a1f2c..3b058ee4fd8 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1609,13 +1609,13 @@ func_exit: order to avoid bogus Valgrind or MSAN warnings.*/ buf_block_t* block = reinterpret_cast(bpage); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_MAKE_DEFINED(block->frame, srv_page_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ btr_search_drop_page_hash_index(block); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(block->frame, srv_page_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ buf_pool_mutex_enter(buf_pool); @@ -1660,9 +1660,9 @@ buf_LRU_block_free_non_file_page( buf_block_set_state(block, BUF_BLOCK_NOT_USED); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(block->frame, srv_page_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ /* Wipe page_no and space_id */ memset(block->frame + FIL_PAGE_OFFSET, 0xfe, 4); memset(block->frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 0xfe, 4); diff --git a/storage/innobase/data/data0data.cc b/storage/innobase/data/data0data.cc index 49bb8715a51..47e58f1614a 100644 --- a/storage/innobase/data/data0data.cc +++ b/storage/innobase/data/data0data.cc @@ -229,7 +229,7 @@ dtuple_validate( const dtuple_t* tuple) /*!< in: tuple */ { ut_ad(tuple->magic_n == DATA_TUPLE_MAGIC_N); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind const ulint n_fields = dtuple_get_n_fields(tuple); for (ulint i = 0; i < n_fields; i++) { @@ -240,7 +240,7 @@ dtuple_validate( dfield_get_len(field)); } } -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ ut_ad(dtuple_check_typed(tuple)); return(TRUE); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 8928e4af5dc..4fa5fa4aa98 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2281,7 +2281,8 @@ fil_check_pending_ops(const fil_space_t* space, ulint count) if (ulint n_pending_ops = my_atomic_loadlint(&space->n_pending_ops)) { - if (count > 5000) { + /* Give a warning every 10 second, starting after 1 second */ + if ((count % 500) == 50) { ib::warn() << "Trying to close/delete/truncate" " tablespace '" << space->name << "' but there are " << n_pending_ops diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index f4d3b49c4a4..46b53d87bf3 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -7334,9 +7334,9 @@ build_template_field( ut_ad(clust_index->table == index->table); templ = prebuilt->mysql_template + prebuilt->n_template++; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(templ, sizeof *templ); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ templ->is_virtual = !field->stored_in_db(); if (!templ->is_virtual) { @@ -8454,9 +8454,9 @@ calc_row_difference( /* The field has changed */ ufield = uvect->fields + n_changed; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(ufield, sizeof *ufield); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ /* Let us use a dummy dfield to make the conversion from the MySQL column format to the InnoDB format */ diff --git a/storage/innobase/include/dict0stats.ic b/storage/innobase/include/dict0stats.ic index 98024935e16..3853af2d409 100644 --- a/storage/innobase/include/dict0stats.ic +++ b/storage/innobase/include/dict0stats.ic @@ -187,7 +187,7 @@ dict_stats_deinit( table->stat_initialized = FALSE; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(&table->stat_n_rows, sizeof table->stat_n_rows); MEM_UNDEFINED(&table->stat_clustered_index_size, sizeof table->stat_clustered_index_size); @@ -220,7 +220,7 @@ dict_stats_deinit( &index->stat_n_leaf_pages, sizeof(index->stat_n_leaf_pages)); } -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ dict_table_stats_unlock(table, RW_X_LATCH); } diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 84e8ece2d77..eaf47789486 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -654,14 +654,14 @@ Use MONITOR_DEC if appropriate mutex protection exists. } \ } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind # define MONITOR_CHECK_DEFINED(value) do { \ mon_type_t m = value; \ MEM_CHECK_DEFINED(&m, sizeof m); \ } while (0) -#else /* HAVE_valgrind_or_MSAN */ +#else /* HAVE_valgrind */ # define MONITOR_CHECK_DEFINED(value) (void) 0 -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ #define MONITOR_INC_VALUE(monitor, value) \ MONITOR_CHECK_DEFINED(value); \ diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 46b2c73cf37..149f3be83d9 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -1248,7 +1248,7 @@ page_cur_insert_rec_low( /* 1. Get the size of the physical record in the page */ rec_size = rec_offs_size(offsets); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind { const void* rec_start = rec - rec_offs_extra_size(offsets); @@ -1263,7 +1263,7 @@ page_cur_insert_rec_low( /* The variable-length header must be valid. */ MEM_CHECK_DEFINED(rec_start, extra_size); } -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ /* 2. Try to find suitable space from page memory management */ @@ -1478,7 +1478,7 @@ page_cur_insert_rec_zip( /* 1. Get the size of the physical record in the page */ rec_size = rec_offs_size(offsets); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind { const void* rec_start = rec - rec_offs_extra_size(offsets); @@ -1493,7 +1493,7 @@ page_cur_insert_rec_zip( /* The variable-length header must be valid. */ MEM_CHECK_DEFINED(rec_start, extra_size); } -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ const bool reorg_before_insert = page_has_garbage(page) && rec_size > page_get_max_insert_size(page, 1) diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index c5b6276caf5..dde46b7aae1 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -998,14 +998,14 @@ exit: goto func_exit; } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(block[i], srv_sort_buf_size); if (crypt_block[i]) { MEM_UNDEFINED(crypt_block[i], srv_sort_buf_size); } -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ } buf[i] = row_merge_buf_empty(buf[i]); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index f659cd4a0a1..adef0e53266 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1243,10 +1243,10 @@ row_ins_foreign_check_on_constraint( update->info_bits = 0; update->n_fields = foreign->n_fields; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(update->fields, update->n_fields * sizeof *update->fields); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ bool affects_fulltext = false; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 97cd7c2a92b..43deb344346 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -372,9 +372,9 @@ row_log_online_op( goto err_exit; } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ ut_ad(log->tail.bytes < srv_sort_buf_size); avail_size = srv_sort_buf_size - log->tail.bytes; @@ -459,10 +459,10 @@ write_failed: index->type |= DICT_CORRUPT; } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.block, srv_sort_buf_size); MEM_UNDEFINED(buf, srv_sort_buf_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); @@ -472,9 +472,9 @@ write_failed: ut_ad(b == log->tail.block + log->tail.bytes); } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ err_exit: mutex_exit(&log->mutex); } @@ -506,9 +506,9 @@ row_log_table_open( { mutex_enter(&log->mutex); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ if (log->error != DB_SUCCESS) { err_exit: @@ -600,10 +600,10 @@ row_log_table_close_func( write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.block, srv_sort_buf_size); MEM_UNDEFINED(buf, srv_sort_buf_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ memcpy(log->tail.block, log->tail.buf + avail, size - avail); log->tail.bytes = size - avail; } else { @@ -612,9 +612,9 @@ write_failed: } log->tail.total += size; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ err_exit: mutex_exit(&log->mutex); @@ -2785,9 +2785,9 @@ row_log_table_apply_ops( ut_ad(new_trx_id_col > 0); ut_ad(new_trx_id_col != ULINT_UNDEFINED); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(&mrec_end, sizeof mrec_end); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ offsets = static_cast(ut_malloc_nokey(i * sizeof *offsets)); rec_offs_set_n_alloc(offsets, i); @@ -3696,9 +3696,9 @@ row_log_apply_ops( ut_ad(!index->is_committed()); ut_ad(rw_lock_own(dict_index_get_lock(index), RW_LOCK_X)); ut_ad(index->online_log); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(&mrec_end, sizeof mrec_end); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ offsets = static_cast(ut_malloc_nokey(i * sizeof *offsets)); rec_offs_set_n_alloc(offsets, i); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 3d21d1d2efc..2bfbbd0ba09 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1027,11 +1027,11 @@ row_merge_buf_write( ut_a(b < &block[srv_sort_buf_size]); ut_a(b == &block[0] + buf->total_size); *b++ = 0; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind /* The rest of the block is uninitialized. Initialize it to avoid bogus warnings. */ memset(b, 0xff, &block[srv_sort_buf_size] - b); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ DBUG_LOG("ib_merge_sort", "write " << reinterpret_cast(b) << ',' << of->fd << ',' << of->offset << " EOF"); @@ -1424,9 +1424,9 @@ row_merge_write_rec( return(NULL); } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(&block[0], srv_sort_buf_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ /* Copy the rest. */ b = &block[0]; @@ -1477,7 +1477,7 @@ row_merge_write_eof( DBUG_RETURN(NULL); } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(&block[0], srv_sort_buf_size); #endif DBUG_RETURN(&block[0]); @@ -2680,10 +2680,10 @@ write_buffers: break; } -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED( &block[0], srv_sort_buf_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ } } merge_buf[i] = row_merge_buf_empty(buf); @@ -3203,9 +3203,9 @@ row_merge( foffs0 = 0; foffs1 = ihalf; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(run_offset, *num_run * sizeof *run_offset); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ for (; foffs0 < ihalf && foffs1 < file->offset; foffs0++, foffs1++) { @@ -3286,9 +3286,9 @@ row_merge( *tmpfd = file->fd; *file = of; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(&block[0], 3 * srv_sort_buf_size); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ return(DB_SUCCESS); } diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index c3dc4e14094..c4c05888eb8 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -982,11 +982,11 @@ row_sel_get_clust_rec( switch (err) { case DB_SUCCESS: case DB_SUCCESS_LOCKED_REC: -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind /* Declare the variable uninitialized. It should be set to DB_SUCCESS at func_exit. */ MEM_UNDEFINED(&err, sizeof err); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ break; default: goto err_exit; @@ -2742,9 +2742,9 @@ row_sel_field_store_in_mysql_format_func( ut_ad(len != UNIV_SQL_NULL); MEM_CHECK_DEFINED(data, len); MEM_CHECK_ADDRESSABLE(dest, templ->mysql_col_len); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(dest, templ->mysql_col_len); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ switch (templ->type) { const byte* field_end; @@ -3653,9 +3653,9 @@ row_sel_copy_cached_field_for_mysql( row_mysql_read_true_varchar( &len, cache, templ->mysql_length_bytes); len += templ->mysql_length_bytes; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(buf, templ->mysql_col_len); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ } else { len = templ->mysql_col_len; } @@ -3724,9 +3724,9 @@ row_sel_dequeue_cached_row_for_mysql( /* The record is long. Copy it field by field, in case there are some long VARCHAR column of which only a small length is being used. */ -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(buf, prebuilt->mysql_prefix_len); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ /* First copy the NULL bits. */ ut_memcpy(buf, cached_rec, prebuilt->null_bitmap_len); @@ -3810,10 +3810,10 @@ row_sel_fetch_last_buf( } ut_ad(prebuilt->fetch_cache_first == 0); -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(prebuilt->fetch_cache[prebuilt->n_fetch_cached], prebuilt->mysql_row_len); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ return(prebuilt->fetch_cache[prebuilt->n_fetch_cached]); } diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index ee432fdddb7..f63347e8589 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1869,9 +1869,9 @@ row_upd_changes_ord_field_binary_func( /* Silence a compiler warning without silencing a Valgrind error. */ dfield_len = 0; -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind MEM_UNDEFINED(&dfield_len, sizeof dfield_len); -#endif /* HAVE_valgrind_or_MSAN */ +#endif /* HAVE_valgrind */ /* See if the column is stored externally. */ buf = row_ext_lookup(ext, col_no, &dfield_len); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 2e761cd7a16..c9b559c7307 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -458,7 +458,7 @@ void trx_free(trx_t*& trx) MEM_UNDEFINED(&trx->state, sizeof trx->state); MEM_UNDEFINED(&trx->mysql_thd, sizeof trx->mysql_thd); #endif -#ifdef HAVE_valgrind_or_MSAN +#ifdef HAVE_valgrind /* Unpoison the memory for innodb_monitor_set_option; it is operating also on the freed transaction objects. We checked that these were initialized in diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index 4a101f7e589..2f13e4cdbb9 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -6123,6 +6123,11 @@ void ha_tokudb::position(const uchar * record) { // memcpy(ref, &key.size, sizeof(uint32_t)); } + /* + tokudb doesn't always write the last byte. Don't that cause problems with + MariaDB + */ + MEM_MAKE_DEFINED(ref, ref_length); TOKUDB_HANDLER_DBUG_VOID_RETURN; } From b6ec1e8bbf0ffca2d715aded694722e0c4b5d484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 2 Jul 2020 16:52:13 +0300 Subject: [PATCH 5/9] MDEV-20377 post-fix: Introduce MEM_MAKE_ADDRESSABLE In AddressSanitizer, we only want memory poisoning to happen in connection with custom memory allocation or freeing. The primary use of MEM_UNDEFINED is for declaring memory uninitialized in Valgrind or MemorySanitizer. We do not want MEM_UNDEFINED to have the unwanted side effect that AddressSanitizer would no longer be able to complain about accessing unallocated memory. MEM_UNDEFINED(): Define as no-op for AddressSanitizer. MEM_MAKE_ADDRESSABLE(): Define as MEM_UNDEFINED() or ASAN_UNPOISON_MEMORY_REGION(). MEM_CHECK_ADDRESSABLE(): Wrap also __asan_region_is_poisoned(). --- include/my_valgrind.h | 29 +++++++++++++++------------ mysys/my_alloc.c | 2 +- sql/table.cc | 1 + storage/innobase/btr/btr0cur.cc | 2 -- storage/innobase/buf/buf0buddy.cc | 2 -- storage/innobase/buf/buf0lru.cc | 8 +------- storage/innobase/handler/ha_innodb.cc | 4 ---- storage/innobase/include/mem0mem.ic | 2 +- storage/innobase/include/ut0pool.h | 14 ++++++------- storage/innobase/os/os0proc.cc | 2 +- storage/innobase/page/page0zip.cc | 4 ++-- storage/innobase/row/row0ins.cc | 2 -- storage/innobase/row/row0log.cc | 18 ++--------------- storage/innobase/row/row0merge.cc | 10 --------- storage/innobase/row/row0sel.cc | 10 --------- storage/innobase/row/row0upd.cc | 2 -- storage/innobase/sync/sync0arr.cc | 2 +- storage/innobase/trx/trx0trx.cc | 8 +++----- 18 files changed, 35 insertions(+), 87 deletions(-) diff --git a/include/my_valgrind.h b/include/my_valgrind.h index c181a8c6b35..62794a2d70c 100644 --- a/include/my_valgrind.h +++ b/include/my_valgrind.h @@ -28,6 +28,7 @@ # include # define HAVE_valgrind # define MEM_UNDEFINED(a,len) __msan_allocated_memory(a,len) +# define MEM_MAKE_ADDRESSABLE(a,len) MEM_UNDEFINED(a,len) # define MEM_MAKE_DEFINED(a,len) __msan_unpoison(a,len) # define MEM_NOACCESS(a,len) ((void) 0) # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0) @@ -38,6 +39,7 @@ #elif defined(HAVE_VALGRIND_MEMCHECK_H) && defined(HAVE_valgrind) # include # define MEM_UNDEFINED(a,len) VALGRIND_MAKE_MEM_UNDEFINED(a,len) +# define MEM_MAKE_ADDRESSABLE(a,len) MEM_UNDEFINED(a,len) # define MEM_MAKE_DEFINED(a,len) VALGRIND_MAKE_MEM_DEFINED(a,len) # define MEM_NOACCESS(a,len) VALGRIND_MAKE_MEM_NOACCESS(a,len) # define MEM_CHECK_ADDRESSABLE(a,len) VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,len) @@ -49,16 +51,19 @@ # include /* How to do manual poisoning: https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ -# define MEM_UNDEFINED(a,len) ASAN_UNPOISON_MEMORY_REGION(a,len) +# define MEM_UNDEFINED(a,len) ((void) 0) +# define MEM_MAKE_ADDRESSABLE(a,len) ASAN_UNPOISON_MEMORY_REGION(a,len) # define MEM_MAKE_DEFINED(a,len) ((void) 0) # define MEM_NOACCESS(a,len) ASAN_POISON_MEMORY_REGION(a,len) -# define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0) +# define MEM_CHECK_ADDRESSABLE(a,len) \ + assert(!__asan_region_is_poisoned((void*) a,len)) # define MEM_CHECK_DEFINED(a,len) ((void) 0) # define MEM_GET_VBITS(a,b,len) ((void) 0) # define MEM_SET_VBITS(a,b,len) ((void) 0) # define REDZONE_SIZE 8 #else -# define MEM_UNDEFINED(a,len) ((void) (a), (void) (len)) +# define MEM_UNDEFINED(a,len) ((void) 0) +# define MEM_MAKE_ADDRESSABLE(a,len) ((void) 0) # define MEM_MAKE_DEFINED(a,len) ((void) 0) # define MEM_NOACCESS(a,len) ((void) 0) # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0) @@ -74,24 +79,22 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ #define IF_VALGRIND(A,B) B #endif - #ifdef TRASH_FREED_MEMORY /* - TRASH_FILL() has to call MEM_UNDEFINED() to cancel any effect of TRASH_FREE(). + _TRASH_FILL() has to call MEM_MAKE_ADDRESSABLE() to cancel any effect of + TRASH_FREE(). This can happen in the case one does TRASH_ALLOC(A,B) ; TRASH_FREE(A,B) ; TRASH_ALLOC(A,B) to reuse the same memory in an internal memory allocator like MEM_ROOT. - For my_malloc() and safemalloc() the extra MEM_UNDEFINED is bit of an - overkill. - TRASH_FILL() is an internal function and should not be used externally. + _TRASH_FILL() is an internal function and should not be used externally. */ -#define TRASH_FILL(A,B,C) do { const size_t trash_tmp= (B); MEM_UNDEFINED(A, trash_tmp); memset(A, C, trash_tmp); } while (0) +#define _TRASH_FILL(A,B,C) do { const size_t trash_tmp= (B); MEM_MAKE_ADDRESSABLE(A, trash_tmp); memset(A, C, trash_tmp); } while (0) #else -#define TRASH_FILL(A,B,C) do { MEM_UNDEFINED((A), (B)); } while (0) +#define _TRASH_FILL(A,B,C) do { MEM_UNDEFINED((A), (B)); } while (0) #endif -/** Note that some memory became allocated or uninitialized. */ -#define TRASH_ALLOC(A,B) do { TRASH_FILL(A,B,0xA5); MEM_UNDEFINED(A,B); } while(0) +/** Note that some memory became allocated and/or uninitialized. */ +#define TRASH_ALLOC(A,B) do { _TRASH_FILL(A,B,0xA5); MEM_MAKE_ADDRESSABLE(A,B); } while(0) /** Note that some memory became freed. (Prohibit further access to it.) */ -#define TRASH_FREE(A,B) do { TRASH_FILL(A,B,0x8F); MEM_NOACCESS(A,B); } while(0) +#define TRASH_FREE(A,B) do { _TRASH_FILL(A,B,0x8F); MEM_NOACCESS(A,B); } while(0) #endif /* MY_VALGRIND_INCLUDED */ diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index b24e9c21623..c5fe75c1410 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -212,7 +212,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) uchar* point; reg1 USED_MEM *next= 0; reg2 USED_MEM **prev; - size_t original_length = length; + size_t original_length __attribute__((unused)) = length; DBUG_ENTER("alloc_root"); DBUG_PRINT("enter",("root: %p name: %s", mem_root, mem_root->name)); DBUG_ASSERT(alloc_root_inited(mem_root)); diff --git a/sql/table.cc b/sql/table.cc index ec2d86e232e..7aa7abfa006 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -1644,6 +1644,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, share->rec_buff_length= rec_buff_length; if (!(record= (uchar *) alloc_root(&share->mem_root, rec_buff_length))) goto err; /* purecov: inspected */ + /* Mark bytes after record as not accessable to catch overrun bugs */ MEM_NOACCESS(record + share->reclength, rec_buff_length - share->reclength); share->default_values= record; memcpy(record, frm_image + record_offset, share->reclength); diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index d9684528195..2d0f92aa499 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1179,12 +1179,10 @@ btr_cur_search_to_nth_level_func( ut_ad(!(index->type & DICT_FTS)); ut_ad(index->page != FIL_NULL); -#ifdef HAVE_valgrind MEM_UNDEFINED(&cursor->up_match, sizeof cursor->up_match); MEM_UNDEFINED(&cursor->up_bytes, sizeof cursor->up_bytes); MEM_UNDEFINED(&cursor->low_match, sizeof cursor->low_match); MEM_UNDEFINED(&cursor->low_bytes, sizeof cursor->low_bytes); -#endif /* HAVE_valgrind */ #ifdef UNIV_DEBUG cursor->up_match = ULINT_UNDEFINED; cursor->low_match = ULINT_UNDEFINED; diff --git a/storage/innobase/buf/buf0buddy.cc b/storage/innobase/buf/buf0buddy.cc index f1849dc28bf..65ef42ef13e 100644 --- a/storage/innobase/buf/buf0buddy.cc +++ b/storage/innobase/buf/buf0buddy.cc @@ -393,9 +393,7 @@ buf_buddy_block_free( HASH_DELETE(buf_page_t, hash, buf_pool->zip_hash, fold, bpage); ut_d(memset(buf, 0, srv_page_size)); -#ifdef HAVE_valgrind MEM_UNDEFINED(buf, srv_page_size); -#endif /* HAVE_valgrind */ block = (buf_block_t*) bpage; buf_page_mutex_enter(block); diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 3b058ee4fd8..bfd0ada1f05 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -806,7 +806,7 @@ buf_LRU_get_free_only( assert_block_ahi_empty(block); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - MEM_UNDEFINED(block->frame, srv_page_size); + MEM_MAKE_ADDRESSABLE(block->frame, srv_page_size); ut_ad(buf_pool_from_block(block) == buf_pool); @@ -1609,13 +1609,9 @@ func_exit: order to avoid bogus Valgrind or MSAN warnings.*/ buf_block_t* block = reinterpret_cast(bpage); -#ifdef HAVE_valgrind MEM_MAKE_DEFINED(block->frame, srv_page_size); -#endif /* HAVE_valgrind */ btr_search_drop_page_hash_index(block); -#ifdef HAVE_valgrind MEM_UNDEFINED(block->frame, srv_page_size); -#endif /* HAVE_valgrind */ buf_pool_mutex_enter(buf_pool); @@ -1660,9 +1656,7 @@ buf_LRU_block_free_non_file_page( buf_block_set_state(block, BUF_BLOCK_NOT_USED); -#ifdef HAVE_valgrind MEM_UNDEFINED(block->frame, srv_page_size); -#endif /* HAVE_valgrind */ /* Wipe page_no and space_id */ memset(block->frame + FIL_PAGE_OFFSET, 0xfe, 4); memset(block->frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 0xfe, 4); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 46b53d87bf3..7234e73f28c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -7334,9 +7334,7 @@ build_template_field( ut_ad(clust_index->table == index->table); templ = prebuilt->mysql_template + prebuilt->n_template++; -#ifdef HAVE_valgrind MEM_UNDEFINED(templ, sizeof *templ); -#endif /* HAVE_valgrind */ templ->is_virtual = !field->stored_in_db(); if (!templ->is_virtual) { @@ -8454,9 +8452,7 @@ calc_row_difference( /* The field has changed */ ufield = uvect->fields + n_changed; -#ifdef HAVE_valgrind MEM_UNDEFINED(ufield, sizeof *ufield); -#endif /* HAVE_valgrind */ /* Let us use a dummy dfield to make the conversion from the MySQL column format to the InnoDB format */ diff --git a/storage/innobase/include/mem0mem.ic b/storage/innobase/include/mem0mem.ic index c1e7348a548..9236bbef05d 100644 --- a/storage/innobase/include/mem0mem.ic +++ b/storage/innobase/include/mem0mem.ic @@ -203,7 +203,7 @@ mem_heap_alloc( mem_block_set_free(block, free + MEM_SPACE_NEEDED(n)); buf = buf + REDZONE_SIZE; - MEM_UNDEFINED(buf, n - REDZONE_SIZE); + MEM_MAKE_ADDRESSABLE(buf, n - REDZONE_SIZE); return(buf); } diff --git a/storage/innobase/include/ut0pool.h b/storage/innobase/include/ut0pool.h index 703a07a23f2..4a5f58f6fae 100644 --- a/storage/innobase/include/ut0pool.h +++ b/storage/innobase/include/ut0pool.h @@ -89,13 +89,12 @@ struct Pool { ut_ad(elem->m_pool == this); #ifdef __SANITIZE_ADDRESS__ /* Unpoison the memory for AddressSanitizer */ - MEM_UNDEFINED(&elem->m_type, sizeof elem->m_type); + MEM_MAKE_ADDRESSABLE(&elem->m_type, + sizeof elem->m_type); #endif -#ifdef HAVE_valgrind - /* Declare the contents as initialized for Valgrind; + /* Declare the contents initialized; we checked this in mem_free(). */ MEM_MAKE_DEFINED(&elem->m_type, sizeof elem->m_type); -#endif Factory::destroy(&elem->m_type); } @@ -134,14 +133,13 @@ struct Pool { if (elem) { # ifdef __SANITIZE_ADDRESS__ /* Unpoison the memory for AddressSanitizer */ - MEM_UNDEFINED(&elem->m_type, sizeof elem->m_type); + MEM_MAKE_ADDRESSABLE(&elem->m_type, + sizeof elem->m_type); # endif -# ifdef HAVE_valgrind - /* Declare the memory initialized for Valgrind. + /* Declare the memory initialized. The trx_t that are released to the pool are actually initialized; we checked that by MEM_CHECK_DEFINED() in mem_free() below. */ -# endif MEM_MAKE_DEFINED(&elem->m_type, sizeof elem->m_type); } #endif diff --git a/storage/innobase/os/os0proc.cc b/storage/innobase/os/os0proc.cc index 508a13de2ca..4a4076f4a1f 100644 --- a/storage/innobase/os/os0proc.cc +++ b/storage/innobase/os/os0proc.cc @@ -162,7 +162,7 @@ os_mem_free_large( // And we must unpoison it by ourself as specified in documentation // for __asan_poison_memory_region() in sanitizer/asan_interface.h // munmap() doesn't do it for us automatically. - MEM_UNDEFINED(ptr, size); + MEM_MAKE_ADDRESSABLE(ptr, size); #endif /* __SANITIZE_ADDRESS__ */ #ifdef HAVE_LINUX_LARGE_PAGES diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index ecfea3a2e90..f1f10bcd58d 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -1577,11 +1577,11 @@ err_exit: ut_ad(buf + c_stream.total_out == c_stream.next_out); ut_ad((ulint) (storage - c_stream.next_out) >= c_stream.avail_out); -#ifdef HAVE_valgrind +#if defined HAVE_valgrind && !__has_feature(memory_sanitizer) /* Valgrind believes that zlib does not initialize some bits in the last 7 or 8 bytes of the stream. Make Valgrind happy. */ MEM_MAKE_DEFINED(buf, c_stream.total_out); -#endif /* HAVE_valgrind */ +#endif /* HAVE_valgrind && !memory_sanitizer */ /* Zero out the area reserved for the modification log. Space for the end marker of the modification log is not diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index adef0e53266..d6596d586ab 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1243,10 +1243,8 @@ row_ins_foreign_check_on_constraint( update->info_bits = 0; update->n_fields = foreign->n_fields; -#ifdef HAVE_valgrind MEM_UNDEFINED(update->fields, update->n_fields * sizeof *update->fields); -#endif /* HAVE_valgrind */ bool affects_fulltext = false; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 43deb344346..a3bf91a1c74 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -372,9 +372,7 @@ row_log_online_op( goto err_exit; } -#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); -#endif /* HAVE_valgrind */ ut_ad(log->tail.bytes < srv_sort_buf_size); avail_size = srv_sort_buf_size - log->tail.bytes; @@ -459,10 +457,8 @@ write_failed: index->type |= DICT_CORRUPT; } -#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.block, srv_sort_buf_size); MEM_UNDEFINED(buf, srv_sort_buf_size); -#endif /* HAVE_valgrind */ memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); @@ -472,9 +468,7 @@ write_failed: ut_ad(b == log->tail.block + log->tail.bytes); } -#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); -#endif /* HAVE_valgrind */ err_exit: mutex_exit(&log->mutex); } @@ -506,9 +500,7 @@ row_log_table_open( { mutex_enter(&log->mutex); -#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); -#endif /* HAVE_valgrind */ if (log->error != DB_SUCCESS) { err_exit: @@ -600,10 +592,9 @@ row_log_table_close_func( write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; } -#ifdef HAVE_valgrind + MEM_UNDEFINED(log->tail.block, srv_sort_buf_size); MEM_UNDEFINED(buf, srv_sort_buf_size); -#endif /* HAVE_valgrind */ memcpy(log->tail.block, log->tail.buf + avail, size - avail); log->tail.bytes = size - avail; } else { @@ -612,9 +603,7 @@ write_failed: } log->tail.total += size; -#ifdef HAVE_valgrind MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); -#endif /* HAVE_valgrind */ err_exit: mutex_exit(&log->mutex); @@ -2785,9 +2774,7 @@ row_log_table_apply_ops( ut_ad(new_trx_id_col > 0); ut_ad(new_trx_id_col != ULINT_UNDEFINED); -#ifdef HAVE_valgrind MEM_UNDEFINED(&mrec_end, sizeof mrec_end); -#endif /* HAVE_valgrind */ offsets = static_cast(ut_malloc_nokey(i * sizeof *offsets)); rec_offs_set_n_alloc(offsets, i); @@ -3696,9 +3683,8 @@ row_log_apply_ops( ut_ad(!index->is_committed()); ut_ad(rw_lock_own(dict_index_get_lock(index), RW_LOCK_X)); ut_ad(index->online_log); -#ifdef HAVE_valgrind + MEM_UNDEFINED(&mrec_end, sizeof mrec_end); -#endif /* HAVE_valgrind */ offsets = static_cast(ut_malloc_nokey(i * sizeof *offsets)); rec_offs_set_n_alloc(offsets, i); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 2bfbbd0ba09..dcc1396d2da 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1424,9 +1424,7 @@ row_merge_write_rec( return(NULL); } -#ifdef HAVE_valgrind MEM_UNDEFINED(&block[0], srv_sort_buf_size); -#endif /* HAVE_valgrind */ /* Copy the rest. */ b = &block[0]; @@ -1477,9 +1475,7 @@ row_merge_write_eof( DBUG_RETURN(NULL); } -#ifdef HAVE_valgrind MEM_UNDEFINED(&block[0], srv_sort_buf_size); -#endif DBUG_RETURN(&block[0]); } @@ -2680,10 +2676,8 @@ write_buffers: break; } -#ifdef HAVE_valgrind MEM_UNDEFINED( &block[0], srv_sort_buf_size); -#endif /* HAVE_valgrind */ } } merge_buf[i] = row_merge_buf_empty(buf); @@ -3203,9 +3197,7 @@ row_merge( foffs0 = 0; foffs1 = ihalf; -#ifdef HAVE_valgrind MEM_UNDEFINED(run_offset, *num_run * sizeof *run_offset); -#endif /* HAVE_valgrind */ for (; foffs0 < ihalf && foffs1 < file->offset; foffs0++, foffs1++) { @@ -3286,9 +3278,7 @@ row_merge( *tmpfd = file->fd; *file = of; -#ifdef HAVE_valgrind MEM_UNDEFINED(&block[0], 3 * srv_sort_buf_size); -#endif /* HAVE_valgrind */ return(DB_SUCCESS); } diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index c4c05888eb8..1dbbc018a9c 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -982,11 +982,9 @@ row_sel_get_clust_rec( switch (err) { case DB_SUCCESS: case DB_SUCCESS_LOCKED_REC: -#ifdef HAVE_valgrind /* Declare the variable uninitialized. It should be set to DB_SUCCESS at func_exit. */ MEM_UNDEFINED(&err, sizeof err); -#endif /* HAVE_valgrind */ break; default: goto err_exit; @@ -2742,9 +2740,7 @@ row_sel_field_store_in_mysql_format_func( ut_ad(len != UNIV_SQL_NULL); MEM_CHECK_DEFINED(data, len); MEM_CHECK_ADDRESSABLE(dest, templ->mysql_col_len); -#ifdef HAVE_valgrind MEM_UNDEFINED(dest, templ->mysql_col_len); -#endif /* HAVE_valgrind */ switch (templ->type) { const byte* field_end; @@ -3653,9 +3649,7 @@ row_sel_copy_cached_field_for_mysql( row_mysql_read_true_varchar( &len, cache, templ->mysql_length_bytes); len += templ->mysql_length_bytes; -#ifdef HAVE_valgrind MEM_UNDEFINED(buf, templ->mysql_col_len); -#endif /* HAVE_valgrind */ } else { len = templ->mysql_col_len; } @@ -3724,9 +3718,7 @@ row_sel_dequeue_cached_row_for_mysql( /* The record is long. Copy it field by field, in case there are some long VARCHAR column of which only a small length is being used. */ -#ifdef HAVE_valgrind MEM_UNDEFINED(buf, prebuilt->mysql_prefix_len); -#endif /* HAVE_valgrind */ /* First copy the NULL bits. */ ut_memcpy(buf, cached_rec, prebuilt->null_bitmap_len); @@ -3810,10 +3802,8 @@ row_sel_fetch_last_buf( } ut_ad(prebuilt->fetch_cache_first == 0); -#ifdef HAVE_valgrind MEM_UNDEFINED(prebuilt->fetch_cache[prebuilt->n_fetch_cached], prebuilt->mysql_row_len); -#endif /* HAVE_valgrind */ return(prebuilt->fetch_cache[prebuilt->n_fetch_cached]); } diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index f63347e8589..0f700c77c36 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1869,9 +1869,7 @@ row_upd_changes_ord_field_binary_func( /* Silence a compiler warning without silencing a Valgrind error. */ dfield_len = 0; -#ifdef HAVE_valgrind MEM_UNDEFINED(&dfield_len, sizeof dfield_len); -#endif /* HAVE_valgrind */ /* See if the column is stored externally. */ buf = row_ext_lookup(ext, col_no, &dfield_len); diff --git a/storage/innobase/sync/sync0arr.cc b/storage/innobase/sync/sync0arr.cc index 65f9353ae77..8e26de10dec 100644 --- a/storage/innobase/sync/sync0arr.cc +++ b/storage/innobase/sync/sync0arr.cc @@ -976,7 +976,7 @@ sync_array_print_long_waits_low( return(false); } -#ifdef HAVE_valgrind +#if defined HAVE_valgrind && !__has_feature(memory_sanitizer) /* Increase the timeouts if running under valgrind because it executes extremely slowly. HAVE_valgrind does not necessary mean that we are running under valgrind but we have no better way to tell. diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index c9b559c7307..e0d8d44a86e 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -453,12 +453,11 @@ void trx_free(trx_t*& trx) #ifdef __SANITIZE_ADDRESS__ /* Unpoison the memory for innodb_monitor_set_option; it is operating also on the freed transaction objects. */ - MEM_UNDEFINED(&trx->mutex, sizeof trx->mutex); + MEM_MAKE_ADDRESSABLE(&trx->mutex, sizeof trx->mutex); /* For innobase_kill_connection() */ - MEM_UNDEFINED(&trx->state, sizeof trx->state); - MEM_UNDEFINED(&trx->mysql_thd, sizeof trx->mysql_thd); + MEM_MAKE_ADDRESSABLE(&trx->state, sizeof trx->state); + MEM_MAKE_ADDRESSABLE(&trx->mysql_thd, sizeof trx->mysql_thd); #endif -#ifdef HAVE_valgrind /* Unpoison the memory for innodb_monitor_set_option; it is operating also on the freed transaction objects. We checked that these were initialized in @@ -467,7 +466,6 @@ void trx_free(trx_t*& trx) /* For innobase_kill_connection() */ MEM_MAKE_DEFINED(&trx->state, sizeof trx->state); MEM_MAKE_DEFINED(&trx->mysql_thd, sizeof trx->mysql_thd); -#endif trx = NULL; } From 53ecc354e398ead01b4dbf2d77170dafc29debd2 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 2 Jul 2020 23:50:56 +0300 Subject: [PATCH 6/9] Fixed errors found by MSAN --- libmysqld/libmysql.c | 1 + mysql-test/main/mysql_upgrade.test | 2 +- mysql-test/suite/csv/read_only.test | 1 + sql/sql_load.cc | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libmysqld/libmysql.c b/libmysqld/libmysql.c index 8c80e1a3f54..c4b3a17f508 100644 --- a/libmysqld/libmysql.c +++ b/libmysqld/libmysql.c @@ -1141,6 +1141,7 @@ my_bool STDCALL mysql_embedded(void) void my_net_local_init(NET *net) { net->max_packet= (uint) net_buffer_length; + net->read_timeout= net->write_timeout= 0; my_net_set_read_timeout(net, CLIENT_NET_READ_TIMEOUT); my_net_set_write_timeout(net, CLIENT_NET_WRITE_TIMEOUT); net->retry_count= 1; diff --git a/mysql-test/main/mysql_upgrade.test b/mysql-test/main/mysql_upgrade.test index 0171fe6c7ba..a3d8ba0375c 100644 --- a/mysql-test/main/mysql_upgrade.test +++ b/mysql-test/main/mysql_upgrade.test @@ -17,7 +17,7 @@ let $MYSQLD_DATADIR= `select @@datadir`; file_exists $MYSQLD_DATADIR/mysql_upgrade_info; --echo Run it again - should say already completed ---replace_result $MYSQL_SERVER_VERSION VERSION +--replace_regex /upgraded to .*, use/upgraded to VERSION, use/ --exec $MYSQL_UPGRADE 2>&1 # It should have created a file in the MySQL Servers datadir diff --git a/mysql-test/suite/csv/read_only.test b/mysql-test/suite/csv/read_only.test index 2af209182d0..a3c851a6a70 100644 --- a/mysql-test/suite/csv/read_only.test +++ b/mysql-test/suite/csv/read_only.test @@ -1,3 +1,4 @@ +--source include/not_as_root.inc # # MDEV-11883 MariaDB crashes with out-of-memory when query information_schema # diff --git a/sql/sql_load.cc b/sql/sql_load.cc index d5692196457..09af120e6cf 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -2031,8 +2031,8 @@ int READ_INFO::read_xml(THD *thd) case '=': /* attribute name end - read the value */ //check for tag field and attribute name - if(!memcmp(tag.c_ptr_safe(), STRING_WITH_LEN("field")) && - !memcmp(attribute.c_ptr_safe(), STRING_WITH_LEN("name"))) + if(!strcmp(tag.c_ptr_safe(), "field") && + !strcmp(attribute.c_ptr_safe(), "name")) { /* this is format xx From 6e81ba0c12d1b94c0c046e1062f06e426036dbb2 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 3 Jul 2020 01:16:31 +0300 Subject: [PATCH 7/9] Don't give errors for default value copy in create_tmp_table --- mysql-test/main/type_temporal_innodb.result | 15 +++++++++++++++ mysql-test/main/type_temporal_innodb.test | 19 +++++++++++++++++++ sql/sql_select.cc | 7 +++++++ 3 files changed, 41 insertions(+) diff --git a/mysql-test/main/type_temporal_innodb.result b/mysql-test/main/type_temporal_innodb.result index b869822722d..c129f4810e2 100644 --- a/mysql-test/main/type_temporal_innodb.result +++ b/mysql-test/main/type_temporal_innodb.result @@ -160,3 +160,18 @@ SELECT * FROM t1 WHERE LEAST( UTC_TIME(), d ); d 2012-12-21 DROP TABLE t1; +# +# MDEV-17969 Assertion `name' failed in THD::push_warning_truncated_value_for_field +# +CREATE TABLE t1 (c1 DATE , c2 TIMESTAMP) ENGINE=InnoDB; +INSERT INTO t1 VALUES ('2006-07-17','0000-00-00 00:00:00'); +CREATE TABLE t2 (pk INT, a1 TIME) Engine=InnoDB; +INSERT INTO t2 VALUES (6,'00:00:00'); +SET SESSION sql_mode= 'strict_all_tables,no_zero_date'; +CREATE TABLE tbl SELECT * FROM t1 WHERE t1.c1 = (SELECT c2 FROM t2 WHERE pk = 6); +ERROR 22007: Truncated incorrect datetime value: '0000-00-00 00:00:00' +DROP TABLE t1,t2; +SET sql_mode=DEFAULT; +# +# End of 10.3 tests +# diff --git a/mysql-test/main/type_temporal_innodb.test b/mysql-test/main/type_temporal_innodb.test index 81f2f586c51..94f61d54634 100644 --- a/mysql-test/main/type_temporal_innodb.test +++ b/mysql-test/main/type_temporal_innodb.test @@ -66,3 +66,22 @@ CREATE TABLE t1 (d DATE) ENGINE=InnoDB; INSERT INTO t1 VALUES ('2012-12-21'); SELECT * FROM t1 WHERE LEAST( UTC_TIME(), d ); DROP TABLE t1; + +--echo # +--echo # MDEV-17969 Assertion `name' failed in THD::push_warning_truncated_value_for_field +--echo # + +CREATE TABLE t1 (c1 DATE , c2 TIMESTAMP) ENGINE=InnoDB; +INSERT INTO t1 VALUES ('2006-07-17','0000-00-00 00:00:00'); +CREATE TABLE t2 (pk INT, a1 TIME) Engine=InnoDB; +INSERT INTO t2 VALUES (6,'00:00:00'); +SET SESSION sql_mode= 'strict_all_tables,no_zero_date'; +--error ER_TRUNCATED_WRONG_VALUE +CREATE TABLE tbl SELECT * FROM t1 WHERE t1.c1 = (SELECT c2 FROM t2 WHERE pk = 6); +# ^^^ there is no column c2 in table t2 +DROP TABLE t1,t2; +SET sql_mode=DEFAULT; + +--echo # +--echo # End of 10.3 tests +--echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 88a1cdedeac..9f5a406507c 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -17460,6 +17460,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, bool using_unique_constraint= false; bool use_packed_rows= false; bool not_all_columns= !(select_options & TMP_TABLE_ALL_COLUMNS); + bool save_abort_on_warning; char *tmpname,path[FN_REFLEN]; uchar *pos, *group_buff, *bitmaps; uchar *null_flags; @@ -17932,6 +17933,11 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, } null_count= (blob_count == 0) ? 1 : 0; hidden_field_count=param->hidden_field_count; + + /* Protect against warnings in field_conv() in the next loop*/ + save_abort_on_warning= thd->abort_on_warning; + thd->abort_on_warning= 0; + for (i=0,reg_field=table->field; i < field_count; i++,reg_field++,recinfo++) { Field *field= *reg_field; @@ -18018,6 +18024,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, bzero(pos, table->s->reclength - (pos - table->record[0])); MEM_CHECK_DEFINED(table->record[0], table->s->reclength); + thd->abort_on_warning= save_abort_on_warning; param->copy_field_end=copy; param->recinfo= recinfo; // Pointer to after last field store_record(table,s->default_values); // Make empty default record From e6595a06d63fb40f81f6ec0313a37931cdf0e9c5 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 3 Jul 2020 01:18:51 +0300 Subject: [PATCH 8/9] Don't give errors for default value copy in create_tmp_table --- sql/sql_select.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index fed76fbd0fd..326c27eff0a 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -18151,6 +18151,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, bool using_unique_constraint= false; bool use_packed_rows= false; bool not_all_columns= !(select_options & TMP_TABLE_ALL_COLUMNS); + bool save_abort_on_warning; char *tmpname,path[FN_REFLEN]; uchar *pos, *group_buff, *bitmaps; uchar *null_flags; @@ -18618,6 +18619,11 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, } null_count= (blob_count == 0) ? 1 : 0; hidden_field_count=param->hidden_field_count; + + /* Protect against warnings in field_conv() in the next loop*/ + save_abort_on_warning= thd->abort_on_warning; + thd->abort_on_warning= 0; + for (i=0,reg_field=table->field; i < field_count; i++,reg_field++,recinfo++) { Field *field= *reg_field; @@ -18704,6 +18710,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, bzero(pos, table->s->reclength - (pos - table->record[0])); MEM_CHECK_DEFINED(table->record[0], table->s->reclength); + thd->abort_on_warning= save_abort_on_warning; param->copy_field_end=copy; param->recinfo= recinfo; // Pointer to after last field store_record(table,s->default_values); // Make empty default record From 484931325e7bcecddc6daf1a55c008ddd67497e4 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 3 Jul 2020 18:37:33 +0300 Subject: [PATCH 9/9] Fix for MSAN from Marko related to buf_pool_resize --- storage/innobase/buf/buf0buf.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index e2425ccb6cc..c9cff9b3295 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1622,6 +1622,8 @@ buf_chunk_init( return(NULL); } + MEM_MAKE_ADDRESSABLE(chunk->mem, chunk->mem_size()); + #ifdef HAVE_LIBNUMA if (srv_numa_interleave) { struct bitmask *numa_mems_allowed = numa_get_mems_allowed(); @@ -2908,6 +2910,9 @@ withdraw_retry: while (chunk < echunk) { buf_block_t* block = chunk->blocks; + MEM_MAKE_ADDRESSABLE(chunk->mem, + chunk->mem_size()); + for (ulint j = chunk->size; j--; block++) { mutex_free(&block->mutex);