From 0f56e21efa68ba3b37d1171d001c21845c3d2b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 16 Mar 2022 11:49:47 +0200 Subject: [PATCH 01/13] MDEV-28091 PERFORMANCE_SCHEMA unit tests fail due to memory misalignment Let us make the mocked-up pfs_malloc() return aligned memory, just like the actual implementation does. --- storage/perfschema/unittest/stub_pfs_global.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/storage/perfschema/unittest/stub_pfs_global.h b/storage/perfschema/unittest/stub_pfs_global.h index 8a1f9216ba2..b7adbe33504 100644 --- a/storage/perfschema/unittest/stub_pfs_global.h +++ b/storage/perfschema/unittest/stub_pfs_global.h @@ -1,4 +1,5 @@ /* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -24,6 +25,9 @@ #include #include #include +#ifdef HAVE_MEMALIGN +# include +#endif bool pfs_initialized= false; @@ -43,7 +47,17 @@ void *pfs_malloc(size_t size, myf) if (--stub_alloc_fails_after_count <= 0) return NULL; +#ifndef PFS_ALIGNEMENT void *ptr= malloc(size); +#elif defined HAVE_MEMALIGN + void *ptr= memalign(PFS_ALIGNEMENT, size); +#elif defined HAVE_ALIGNED_MALLOC + void *ptr= _aligned_malloc(size, PFS_ALIGNEMENT); +#else + void *ptr; + if (posix_memalign(&ptr, PFS_ALIGNEMENT, size)) + ptr= NULL; +#endif if (ptr != NULL) memset(ptr, 0, size); return ptr; From 75e39f3cba01f9f3a835d8e311658bcbf0453d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 17 Mar 2022 10:13:50 +0200 Subject: [PATCH 02/13] Fix gcc-12 -O2 -Wmaybe-uninitialized --- storage/innobase/fil/fil0fil.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 8075defac4c..6980078f87d 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2021, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2021, MariaDB Corporation. +Copyright (c) 2014, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3316,10 +3316,11 @@ fil_make_filepath( if (path != NULL) { memcpy(full_name, path, path_len); len = path_len; - full_name[len] = '\0'; - os_normalize_path(full_name); } + full_name[len] = '\0'; + os_normalize_path(full_name); + if (trim_name) { /* Find the offset of the last DIR separator and set it to null in order to strip off the old basename from this path. */ From 118826d1734bc4f650f9ec96b3d0d885eedba9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 17 Mar 2022 10:20:07 +0200 Subject: [PATCH 03/13] Fix gcc-12 -O2 -Warray-bounds --- sql/handler.h | 4 +-- sql/sql_table.cc | 43 +++++++++++++++++---------------- storage/innobase/fts/fts0fts.cc | 4 +-- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/sql/handler.h b/sql/handler.h index 02a4a76c6c1..27836f1735f 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -2,7 +2,7 @@ #define HANDLER_INCLUDED /* Copyright (c) 2000, 2019, Oracle and/or its affiliates. - Copyright (c) 2009, 2021, MariaDB + Copyright (c) 2009, 2022, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -4275,7 +4275,7 @@ static inline const char *ha_resolve_storage_engine_name(const handlerton *db_ty static inline bool ha_check_storage_engine_flag(const handlerton *db_type, uint32 flag) { - return db_type == NULL ? FALSE : MY_TEST(db_type->flags & flag); + return db_type && (db_type->flags & flag); } static inline bool ha_storage_engine_is_enabled(const handlerton *db_type) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index e1f752191ae..a2dc5c97aeb 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -9191,22 +9191,24 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, create_info->used_fields |= HA_CREATE_USED_ROW_FORMAT; } + handlerton * const old_db_type= table->s->db_type(); + handlerton *new_db_type= create_info->db_type; + DBUG_PRINT("info", ("old type: %s new type: %s", - ha_resolve_storage_engine_name(table->s->db_type()), - ha_resolve_storage_engine_name(create_info->db_type))); - if (ha_check_storage_engine_flag(table->s->db_type(), HTON_ALTER_NOT_SUPPORTED)) + ha_resolve_storage_engine_name(old_db_type), + ha_resolve_storage_engine_name(new_db_type))); + if (ha_check_storage_engine_flag(old_db_type, HTON_ALTER_NOT_SUPPORTED)) { DBUG_PRINT("info", ("doesn't support alter")); - my_error(ER_ILLEGAL_HA, MYF(0), hton_name(table->s->db_type())->str, + my_error(ER_ILLEGAL_HA, MYF(0), hton_name(old_db_type)->str, alter_ctx.db, alter_ctx.table_name); DBUG_RETURN(true); } - if (ha_check_storage_engine_flag(create_info->db_type, - HTON_ALTER_NOT_SUPPORTED)) + if (ha_check_storage_engine_flag(new_db_type, HTON_ALTER_NOT_SUPPORTED)) { DBUG_PRINT("info", ("doesn't support alter")); - my_error(ER_ILLEGAL_HA, MYF(0), hton_name(create_info->db_type)->str, + my_error(ER_ILLEGAL_HA, MYF(0), hton_name(new_db_type)->str, alter_ctx.new_db, alter_ctx.new_name); DBUG_RETURN(true); } @@ -9349,6 +9351,17 @@ do_continue:; DBUG_RETURN(true); } } + /* + If the old table had partitions and we are doing ALTER TABLE ... + engine= , the new table must preserve the original + partitioning. This means that the new engine is still the + partitioning engine, not the engine specified in the parser. + This is discovered in prep_alter_part_table, which in such case + updates create_info->db_type. + It's therefore important that the assignment below is done + after prep_alter_part_table. + */ + new_db_type= create_info->db_type; #endif if (mysql_prepare_alter_table(thd, table, create_info, alter_info, @@ -9424,7 +9437,7 @@ do_continue:; Alter_info::ALTER_TABLE_ALGORITHM_INPLACE) || is_inplace_alter_impossible(table, create_info, alter_info) || IF_PARTITIONING((partition_changed && - !(table->s->db_type()->partition_flags() & HA_USE_AUTO_PARTITION)), 0)) + !(old_db_type->partition_flags() & HA_USE_AUTO_PARTITION)), 0)) { if (alter_info->requested_algorithm == Alter_info::ALTER_TABLE_ALGORITHM_INPLACE) @@ -9441,22 +9454,10 @@ do_continue:; request table rebuild. Set ALTER_RECREATE flag to force table rebuild. */ - if (create_info->db_type == table->s->db_type() && + if (new_db_type == old_db_type && create_info->used_fields & HA_CREATE_USED_ENGINE) alter_info->flags|= Alter_info::ALTER_RECREATE; - /* - If the old table had partitions and we are doing ALTER TABLE ... - engine= , the new table must preserve the original - partitioning. This means that the new engine is still the - partitioning engine, not the engine specified in the parser. - This is discovered in prep_alter_part_table, which in such case - updates create_info->db_type. - It's therefore important that the assignment below is done - after prep_alter_part_table. - */ - handlerton *new_db_type= create_info->db_type; - handlerton *old_db_type= table->s->db_type(); TABLE *new_table= NULL; ha_rows copied=0,deleted=0; diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 3cb15d64e91..6f9349a9c35 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -2313,9 +2313,7 @@ fts_trx_table_create( fts_trx_table_t* ftt; ftt = static_cast( - mem_heap_alloc(fts_trx->heap, sizeof(*ftt))); - - memset(ftt, 0x0, sizeof(*ftt)); + mem_heap_zalloc(fts_trx->heap, sizeof *ftt)); ftt->table = table; ftt->fts_trx = fts_trx; From 22fd31c5883622b5c7451cee74bc5d087d81e112 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 16 Mar 2022 14:37:55 +0400 Subject: [PATCH 04/13] MDEV-28078 Garbage on multiple equal ENUMs with tricky character sets TYPELIBs for ENUM/SET columns could erroneously undergo redundant hex-unescaping at the table open time. Fix: - Prevent multiple unescaping of the same TYPELIB - Prevent sharing TYPELIBs between columns with different mbminlen --- mysql-test/r/ctype_utf32.result | 25 +++++++++++++++++++++++++ mysql-test/t/ctype_utf32.test | 19 +++++++++++++++++++ sql/table.cc | 20 ++++++++++++++++++-- sql/unireg.cc | 11 ++++++++++- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result index 143fff9e419..22cea274182 100644 --- a/mysql-test/r/ctype_utf32.result +++ b/mysql-test/r/ctype_utf32.result @@ -2913,5 +2913,30 @@ t1 CREATE TABLE `t1` ( DROP TABLE t1; SET NAMES utf8; # +# MDEV-28078 Garbage on multiple equal ENUMs with tricky character sets +# +CREATE TABLE t1 ( +c1 ENUM ('a','b') CHARACTER SET utf32 DEFAULT 'a', +c2 ENUM ('a','b') CHARACTER SET utf32 DEFAULT 'a' +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` enum('a','b') CHARACTER SET utf32 DEFAULT 'a', + `c2` enum('a','b') CHARACTER SET utf32 DEFAULT 'a' +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 ENUM ('00000061','00000062') DEFAULT '00000061' COLLATE latin1_bin, +c2 ENUM ('a','b') DEFAULT 'a' COLLATE utf32_general_ci +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` enum('00000061','00000062') CHARACTER SET latin1 COLLATE latin1_bin DEFAULT '00000061', + `c2` enum('a','b') CHARACTER SET utf32 DEFAULT 'a' +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +# # End of 10.2 tests # diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test index 46ff333b5f7..739096ae9cb 100644 --- a/mysql-test/t/ctype_utf32.test +++ b/mysql-test/t/ctype_utf32.test @@ -1067,6 +1067,25 @@ DROP TABLE t1; SET NAMES utf8; +--echo # +--echo # MDEV-28078 Garbage on multiple equal ENUMs with tricky character sets +--echo # + +CREATE TABLE t1 ( + c1 ENUM ('a','b') CHARACTER SET utf32 DEFAULT 'a', + c2 ENUM ('a','b') CHARACTER SET utf32 DEFAULT 'a' +); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + c1 ENUM ('00000061','00000062') DEFAULT '00000061' COLLATE latin1_bin, + c2 ENUM ('a','b') DEFAULT 'a' COLLATE utf32_general_ci +); +SHOW CREATE TABLE t1; +DROP TABLE t1; + + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/table.cc b/sql/table.cc index ca6ce02e4f2..1f7b6452303 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -1229,6 +1229,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, plugin_ref se_plugin= 0; MEM_ROOT *old_root= thd->mem_root; Virtual_column_info **table_check_constraints; + bool *interval_unescaped= NULL; DBUG_ENTER("TABLE_SHARE::init_from_binary_frm_image"); keyinfo= &first_keyinfo; @@ -1686,6 +1687,13 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, goto err; + if (interval_count) + { + if (!(interval_unescaped= (bool*) my_alloca(interval_count * sizeof(bool)))) + goto err; + bzero(interval_unescaped, interval_count * sizeof(bool)); + } + field_ptr= share->field; table_check_constraints= share->check_constraints; read_length=(uint) (share->fields * field_pack_length + @@ -1956,11 +1964,17 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, if (share->mysql_version < 100200) pack_flag&= ~FIELDFLAG_LONG_DECIMAL; - if (interval_nr && charset->mbminlen > 1) + if (interval_nr && charset->mbminlen > 1 && + !interval_unescaped[interval_nr - 1]) { - /* Unescape UCS2 intervals from HEX notation */ + /* + Unescape UCS2/UTF16/UTF32 intervals from HEX notation. + Note, ENUM/SET columns with equal value list share a single + copy of TYPELIB. Unescape every TYPELIB only once. + */ TYPELIB *interval= share->intervals + interval_nr - 1; unhex_type2(interval); + interval_unescaped[interval_nr - 1]= true; } #ifndef TO_BE_DELETED_ON_PRODUCTION @@ -2610,6 +2624,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, share->error= OPEN_FRM_OK; thd->status_var.opened_shares++; thd->mem_root= old_root; + my_afree(interval_unescaped); DBUG_RETURN(0); err: @@ -2623,6 +2638,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, open_table_error(share, OPEN_FRM_CORRUPTED, share->open_errno); thd->mem_root= old_root; + my_afree(interval_unescaped); DBUG_RETURN(HA_ERR_NOT_A_TABLE); } diff --git a/sql/unireg.cc b/sql/unireg.cc index 7974255af35..5471290651b 100644 --- a/sql/unireg.cc +++ b/sql/unireg.cc @@ -757,7 +757,16 @@ static uint get_interval_id(uint *int_count,List &create_fields, while ((field=it++) != last_field) { - if (field->interval_id && field->interval->count == interval->count) + /* + ENUM/SET columns with equal value lists share a single + copy of the underlying TYPELIB. + Fields with different mbminlen can't reuse TYPELIBs, because: + - mbminlen==1 are written to FRM as is + - mbminlen>1 are written to FRM in hex-encoded format + */ + if (field->interval_id && + field->interval->count == interval->count && + field->charset->mbminlen == last_field->charset->mbminlen) { const char **a,**b; for (a=field->interval->type_names, b=interval->type_names ; From 74e668eaeb4271845fd69d2945085383c431e333 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 17 Mar 2022 16:58:43 +0200 Subject: [PATCH 05/13] Fixed warning for maria.maria-recovery2 about crashed table The bug was a missing va_start in eprint() which caused a wrong table name to be printed. Patch backported from 10.3. --- storage/maria/ma_recovery_util.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/storage/maria/ma_recovery_util.c b/storage/maria/ma_recovery_util.c index 9443ba90f6c..cb4203392dd 100644 --- a/storage/maria/ma_recovery_util.c +++ b/storage/maria/ma_recovery_util.c @@ -59,11 +59,12 @@ void tprint(FILE *trace_file __attribute__ ((unused)), va_list args; #ifndef DBUG_OFF { - char buff[1024], *end; + char buff[1024]; + size_t length; va_start(args, format); - vsnprintf(buff, sizeof(buff)-1, format, args); - if (*(end= strend(buff)) == '\n') - *end= 0; /* Don't print end \n */ + length= my_vsnprintf(buff, sizeof(buff)-1, format, args); + if (length && buff[length-1] == '\n') + buff[length-1]= 0; /* Don't print end \n */ DBUG_PRINT("info", ("%s", buff)); va_end(args); } @@ -95,6 +96,7 @@ void eprint(FILE *trace_file __attribute__ ((unused)), fputc('\n', trace_file); if (trace_file != stderr) { + va_start(args, format); my_printv_error(HA_ERR_INITIALIZATION, format, MYF(0), args); } va_end(args); From 421153848f0002c9e9b5e204d4c6e85dfcfb5600 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 20 Mar 2022 21:36:41 +0100 Subject: [PATCH 06/13] MDEV-27980 file-key-management plugin disabled in mysql_install_db breaks automated deployments (and container initialization) Revert "Silence the file-key-management plugin during mysql_install_db" This reverts commit e99d3da6381023395c86f679bb76b00b4385dc2d. --- scripts/mysql_install_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 3df48f0eb95..bf2e2617b9b 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -506,7 +506,7 @@ mysqld_install_cmd_line() { "$mysqld_bootstrap" $defaults $defaults_group_suffix "$mysqld_opt" --bootstrap $silent_startup\ "--basedir=$basedir" "--datadir=$ldata" --log-warnings=0 --enforce-storage-engine="" \ - "--plugin-dir=${plugindir}" --loose-disable-plugin-file-key-management \ + "--plugin-dir=${plugindir}" \ $args --max_allowed_packet=8M \ --net_buffer_length=16K } From f54d6380d2662d7bc7f173bf96f5dc3d7cf3aec1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 21 Mar 2022 11:01:40 +0100 Subject: [PATCH 07/13] MDEV-27980 file-key-management plugin disabled in mysql_install_db breaks automated deployments (and container initialization) fix a 2015 typo in build scripts. --without-plugin=plugin_file_key_management translates to -DPLUGIN_PLUGIN_FILE_KEY_MANAGEMENT=NO replace it with a line from 10.4 that builds the plugin dynamically. --- BUILD/SETUP.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD/SETUP.sh b/BUILD/SETUP.sh index 8a7238c359f..77504522f11 100755 --- a/BUILD/SETUP.sh +++ b/BUILD/SETUP.sh @@ -211,7 +211,7 @@ fi max_no_embedded_configs="$SSL_LIBRARY --with-plugins=max" max_no_qc_configs="$SSL_LIBRARY --with-plugins=max --without-query-cache" -max_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server --with-libevent --without-plugin=plugin_file_key_management --with-plugin-rocksdb=dynamic --with-plugin-test_sql_discovery=DYNAMIC" +max_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server --with-libevent --with-plugin-rocksdb=dynamic --with-plugin-test_sql_discovery=DYNAMIC --with-plugin-file_key_management=DYNAMIC" all_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server --with-innodb_plugin --with-libevent" # From 6277e7df6b84f6d5931dab66d3edf8859d5b16d3 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 22 Nov 2021 09:58:46 +0400 Subject: [PATCH 08/13] MDEV-22742 UBSAN: Many overflow issues in strings/decimal.c - runtime error: signed integer overflow: x * y cannot be represented in type 'long long int' (on optimized builds). Avoid integer overflow, do the check before the calculation. --- strings/decimal.c | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/strings/decimal.c b/strings/decimal.c index 9d18a9ce52a..78b4c12b0ef 100644 --- a/strings/decimal.c +++ b/strings/decimal.c @@ -1128,13 +1128,21 @@ int decimal2ulonglong(const decimal_t *from, ulonglong *to) for (intg=from->intg; intg > 0; intg-=DIG_PER_DEC1) { - ulonglong y=x; - x=x*DIG_BASE + *buf++; - if (unlikely(y > ((ulonglong) ULONGLONG_MAX/DIG_BASE) || x < y)) + /* + Check that the decimal is bigger than any possible integer. + Do it before we do the x*=DIB_BASE to avoid integer + overflow. + */ + if (unlikely ( + x >= ULONGLONG_MAX/DIG_BASE && + (x > ULONGLONG_MAX/DIG_BASE || + *buf > (dec1) (ULONGLONG_MAX%DIG_BASE)))) { *to=ULONGLONG_MAX; return E_DEC_OVERFLOW; } + + x=x*DIG_BASE + *buf++; } *to=x; for (frac=from->frac; unlikely(frac > 0); frac-=DIG_PER_DEC1) @@ -1151,23 +1159,29 @@ int decimal2longlong(const decimal_t *from, longlong *to) for (intg=from->intg; intg > 0; intg-=DIG_PER_DEC1) { - longlong y=x; /* + Check that the decimal is less than any possible integer. + Do it before we do the x*=DIB_BASE to avoid integer + overflow. Attention: trick! we're calculating -|from| instead of |from| here because |LONGLONG_MIN| > LONGLONG_MAX - so we can convert -9223372036854775808 correctly + so we can convert -9223372036854775808 correctly. */ - x=x*DIG_BASE - *buf++; - if (unlikely(y < (LONGLONG_MIN/DIG_BASE) || x > y)) + if (unlikely ( + x <= LONGLONG_MIN/DIG_BASE && + (x < LONGLONG_MIN/DIG_BASE || + *buf > (dec1) (-(LONGLONG_MIN%DIG_BASE))))) { /* - the decimal is bigger than any possible integer - return border integer depending on the sign + the decimal is bigger than any possible integer + return border integer depending on the sign */ *to= from->sign ? LONGLONG_MIN : LONGLONG_MAX; return E_DEC_OVERFLOW; } + + x=x*DIG_BASE - *buf++; } /* boundary case: 9223372036854775808 */ if (unlikely(from->sign==0 && x == LONGLONG_MIN)) From 8153c974e60901f5f029b925de75afa3a68f3066 Mon Sep 17 00:00:00 2001 From: Ian Gilfillan Date: Tue, 22 Mar 2022 14:45:55 +0200 Subject: [PATCH 09/13] Update contributors --- CREDITS | 2 ++ mysql-test/r/contributors.result | 2 ++ sql/contributors.h | 2 ++ 3 files changed, 6 insertions(+) diff --git a/CREDITS b/CREDITS index f5e87e18752..35092602ccf 100644 --- a/CREDITS +++ b/CREDITS @@ -4,9 +4,11 @@ organization registered in the USA. The current main sponsors of the MariaDB Foundation are: Alibaba Cloud https://www.alibabacloud.com/ (2017) +Intel https://www.intel.com (2022) MariaDB Corporation https://www.mariadb.com (2013) Microsoft https://microsoft.com/ (2017) ServiceNow https://servicenow.com (2019) +SIT https://sit.org (2022) Tencent Cloud https://cloud.tencent.com (2017) Development Bank of Singapore https://dbs.com (2016) IBM https://www.ibm.com (2017) diff --git a/mysql-test/r/contributors.result b/mysql-test/r/contributors.result index 0c7ca03a2c5..8d72373696c 100644 --- a/mysql-test/r/contributors.result +++ b/mysql-test/r/contributors.result @@ -5,6 +5,8 @@ Tencent Cloud https://cloud.tencent.com Platinum Sponsor of the MariaDB Foundati Microsoft https://microsoft.com/ Platinum Sponsor of the MariaDB Foundation MariaDB Corporation https://mariadb.com Founding member, Platinum Sponsor of the MariaDB Foundation ServiceNow https://servicenow.com Platinum Sponsor of the MariaDB Foundation +Intel https://www.intel.com Platinum Sponsor of the MariaDB Foundation +SIT https://sit.org Platinum Sponsor of the MariaDB Foundation Visma https://visma.com Gold Sponsor of the MariaDB Foundation DBS https://dbs.com Gold Sponsor of the MariaDB Foundation IBM https://www.ibm.com Gold Sponsor of the MariaDB Foundation diff --git a/sql/contributors.h b/sql/contributors.h index e16448ee985..bc8ba4eabbb 100644 --- a/sql/contributors.h +++ b/sql/contributors.h @@ -42,6 +42,8 @@ struct show_table_contributors_st show_table_contributors[]= { {"Microsoft", "https://microsoft.com/", "Platinum Sponsor of the MariaDB Foundation"}, {"MariaDB Corporation", "https://mariadb.com", "Founding member, Platinum Sponsor of the MariaDB Foundation"}, {"ServiceNow", "https://servicenow.com", "Platinum Sponsor of the MariaDB Foundation"}, + {"Intel", "https://www.intel.com", "Platinum Sponsor of the MariaDB Foundation"}, + {"SIT", "https://sit.org", "Platinum Sponsor of the MariaDB Foundation"}, {"Visma", "https://visma.com", "Gold Sponsor of the MariaDB Foundation"}, {"DBS", "https://dbs.com", "Gold Sponsor of the MariaDB Foundation"}, {"IBM", "https://www.ibm.com", "Gold Sponsor of the MariaDB Foundation"}, From 174f1734a9672c13928dfd03d49d25c4325f0ce3 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Wed, 22 Sep 2021 11:25:52 -0600 Subject: [PATCH 10/13] MDEV-14608: mysqlbinlog lastest backupfile size is 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: ======== When using mariadb-binlog with --raw and --stop-never, events from the master's currently active log file should be written to their respective log file specified by --result-file, and shown on-disk. There is a bug where mariadb-binlog does not flush the result file to disk when new events are received Solution: ======== Add a function call to flush mariadb-binlog’s result file after receiving an event in --raw mode. Reviewed By: ============ Andrei Elkin --- client/mysqlbinlog.cc | 1 + .../r/binlog_mysqlbinlog_raw_flush.result | 7 +++ .../t/binlog_mysqlbinlog_raw_flush.test | 45 +++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 mysql-test/suite/binlog/r/binlog_mysqlbinlog_raw_flush.result create mode 100644 mysql-test/suite/binlog/t/binlog_mysqlbinlog_raw_flush.test diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 4e28876cdf6..c7c71f6b00d 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -2510,6 +2510,7 @@ static Exit_status handle_event_raw_mode(PRINT_EVENT_INFO *print_event_info, error("Could not write into log file '%s'", out_file_name); DBUG_RETURN(ERROR_STOP); } + fflush(result_file); DBUG_RETURN(OK_CONTINUE); } diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_raw_flush.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_raw_flush.result new file mode 100644 index 00000000000..9148f0e8c2b --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_raw_flush.result @@ -0,0 +1,7 @@ +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 VALUES (1); +# timeout TIMEOUT MYSQL_BINLOG --raw --read-from-remote-server --user=root --host=127.0.0.1 --port=MASTER_MYPORT --stop-never --result-file=MYSQLTEST_VARDIR/tmp/ master-bin.000001 +# MYSQL_BINLOG MYSQLTEST_VARDIR/tmp/master-bin.000002 > MYSQLTEST_VARDIR/tmp/local-bin.000002.out +FOUND 1 /GTID 0-1-2/ in local-bin.000002.out +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_raw_flush.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_raw_flush.test new file mode 100644 index 00000000000..f95fc0137a2 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_raw_flush.test @@ -0,0 +1,45 @@ +# +# Purpose: +# When using mariadb-binlog with options for --raw and --stop-never, events +# from the master's currently active log file should be written to their +# respective log file specified by --result-file, and shown on-disk. This test +# ensures that the log files on disk, created by mariadb-binlog, have the most +# up-to-date events from the master. +# +# Methodology: +# On the master, rotate to a newly active binlog file and write an event to +# it. Read the master's binlog using mariadb-binlog with --raw and --stop-never +# and write the data to an intermediary binlog file (a timeout is used on this +# command to ensure it exits). Read the local intermediary binlog file to ensure +# that the master's most recent event exists in the local file. +# +# References: +# MDEV-14608: mysqlbinlog lastest backupfile size is 0 +# + +--source include/linux.inc +--source include/have_log_bin.inc + +# Create newly active log +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 VALUES (1); + +# Read binlog data from master to intermediary result file +--let TIMEOUT=1 +--echo # timeout TIMEOUT MYSQL_BINLOG --raw --read-from-remote-server --user=root --host=127.0.0.1 --port=MASTER_MYPORT --stop-never --result-file=MYSQLTEST_VARDIR/tmp/ master-bin.000001 +--error 124 # Error 124 means timeout was reached +--exec timeout $TIMEOUT $MYSQL_BINLOG --raw --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT --stop-never --result-file=$MYSQLTEST_VARDIR/tmp/ master-bin.000001 + +# Ensure the binlog output has the most recent events from the master +--echo # MYSQL_BINLOG MYSQLTEST_VARDIR/tmp/master-bin.000002 > MYSQLTEST_VARDIR/tmp/local-bin.000002.out +--exec $MYSQL_BINLOG $MYSQLTEST_VARDIR/tmp/master-bin.000002 > $MYSQLTEST_VARDIR/tmp/local-bin.000002.out +--let SEARCH_PATTERN= GTID 0-1-2 +--let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/local-bin.000002.out +--source include/search_pattern_in_file.inc + +# Cleanup +DROP TABLE t1; +--remove_file $MYSQLTEST_VARDIR/tmp/master-bin.000001 +--remove_file $MYSQLTEST_VARDIR/tmp/master-bin.000002 +--remove_file $MYSQLTEST_VARDIR/tmp/local-bin.000002.out From 9f4ba624e2f7ad6cd35c842dbb07605f0751f4aa Mon Sep 17 00:00:00 2001 From: Sachin Kumar Date: Mon, 24 May 2021 11:23:03 +0100 Subject: [PATCH 11/13] MDEV-24667 LOAD DATA INFILE on temporary table not written to slave binlog Problem: In regular replication, when master binlogged using statement format slave might not have written an event to its binary log when the Query event aimed at a temporary table. Specifically this was observed with LOAD DATA INFILE. This effect was possible because unlike master slave holds temporary tables in its pool and the master side check of existence of a temporary table at the format bin-logging decision did not apply. Solution: replace THD::has_thd_temporary_tables() with THD::has_temporary_tables which allows to identify temporary table presence on either side. -- Reviewed by Andrei Elkin. --- mysql-test/suite/rpl/r/mdev_24667.result | 30 +++++++++++++ mysql-test/suite/rpl/t/mdev_24667.cnf | 8 ++++ mysql-test/suite/rpl/t/mdev_24667.test | 56 ++++++++++++++++++++++++ sql/sql_class.h | 4 +- sql/temporary_tables.cc | 2 +- 5 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 mysql-test/suite/rpl/r/mdev_24667.result create mode 100644 mysql-test/suite/rpl/t/mdev_24667.cnf create mode 100644 mysql-test/suite/rpl/t/mdev_24667.test diff --git a/mysql-test/suite/rpl/r/mdev_24667.result b/mysql-test/suite/rpl/r/mdev_24667.result new file mode 100644 index 00000000000..7c7342d63d6 --- /dev/null +++ b/mysql-test/suite/rpl/r/mdev_24667.result @@ -0,0 +1,30 @@ +include/rpl_init.inc [topology=1->2->3] +call mtr.add_suppression('Unsafe statement written to the binary log using '); +connection server_1; +set binlog_format=statement; +#first bug +create table t1 (a int); +create temporary table tmp like t1; +load data local infile 'MYSQLTEST_VARDIR/load_data' INTO TABLE tmp; +insert into t1 select * from tmp; +#second bug +create table t2 (a int); +create temporary table tmp2 like t2; +insert into tmp2 values(10); +update tmp2 set a = 20 limit 1; +Warnings: +Note 1592 Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. The statement is unsafe because it uses a LIMIT clause. This is unsafe because the set of rows included cannot be predicted +insert into t2 select * from tmp2; +connection server_2; +connection server_3; +#t1 should have 2 rows +select count(*) = 2 from t1; +count(*) = 2 +1 +#t2 should have 1 rows with a = 20 +select * from t2; +a +20 +connection server_1; +drop table t1, t2, tmp, tmp2; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/mdev_24667.cnf b/mysql-test/suite/rpl/t/mdev_24667.cnf new file mode 100644 index 00000000000..58b605ad928 --- /dev/null +++ b/mysql-test/suite/rpl/t/mdev_24667.cnf @@ -0,0 +1,8 @@ +!include ../my.cnf + +[mysqld.3] +log-slave-updates + +[ENV] +SERVER_MYPORT_3= @mysqld.3.port +SERVER_MYSOCK_3= @mysqld.3.socket diff --git a/mysql-test/suite/rpl/t/mdev_24667.test b/mysql-test/suite/rpl/t/mdev_24667.test new file mode 100644 index 00000000000..d8490b335db --- /dev/null +++ b/mysql-test/suite/rpl/t/mdev_24667.test @@ -0,0 +1,56 @@ +# +# MDEV-24667 LOAD DATA INFILE/inserted rows not written to binlog +# +# In this test we will have a replication configuration like 1->2->3 +# 1 will have statement format +# 2 and 3 will have mixed format +# We will make some updates on temporary table which are unsafe , So 2 must +# Log these queries in row format, Since it is on tmp table , It wont be logged +# So the next query which copies the data from tmp table to normal must be logged +# into the row format. Instead of checking for the binlog We will compare the +# results on the 3, If no binlog is lost(ie it is logged into row format), There +# should not be any data loss. +--let $rpl_topology=1->2->3 +--source include/rpl_init.inc +--source include/have_binlog_format_mixed.inc +call mtr.add_suppression('Unsafe statement written to the binary log using '); +--connection server_1 + +set binlog_format=statement; +--echo #first bug +create table t1 (a int); +create temporary table tmp like t1; +--write_file $MYSQLTEST_VARDIR/load_data +1 +2 +EOF +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +eval load data local infile '$MYSQLTEST_VARDIR/load_data' INTO TABLE tmp; +insert into t1 select * from tmp; + +--echo #second bug +create table t2 (a int); +#insert into t2 values(10); +create temporary table tmp2 like t2; +insert into tmp2 values(10); +update tmp2 set a = 20 limit 1; +insert into t2 select * from tmp2; +--save_master_pos + +--connection server_2 +--sync_with_master +--save_master_pos + +--connection server_3 +--sync_with_master +--echo #t1 should have 2 rows +select count(*) = 2 from t1; +--echo #t2 should have 1 rows with a = 20 +select * from t2; + + +# cleanup +--connection server_1 +drop table t1, t2, tmp, tmp2; +--remove_file $MYSQLTEST_VARDIR/load_data +--source include/rpl_end.inc diff --git a/sql/sql_class.h b/sql/sql_class.h index a748def9b56..d3d54e11671 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3960,13 +3960,13 @@ public: */ DBUG_PRINT("debug", ("temporary_tables: %s, in_sub_stmt: %s, system_thread: %s", - YESNO(has_thd_temporary_tables()), YESNO(in_sub_stmt), + YESNO(has_temporary_tables()), YESNO(in_sub_stmt), show_system_thread(system_thread))); if (in_sub_stmt == 0) { if (wsrep_binlog_format() == BINLOG_FORMAT_ROW) set_current_stmt_binlog_format_row(); - else if (!has_thd_temporary_tables()) + else if (!has_temporary_tables()) set_current_stmt_binlog_format_stmt(); } DBUG_VOID_RETURN; diff --git a/sql/temporary_tables.cc b/sql/temporary_tables.cc index 005a520ff64..fb28ac40aa6 100644 --- a/sql/temporary_tables.cc +++ b/sql/temporary_tables.cc @@ -866,7 +866,7 @@ void THD::restore_tmp_table_share(TMP_TABLE_SHARE *share) @return false Temporary tables exist true No temporary table exist */ -inline bool THD::has_temporary_tables() +bool THD::has_temporary_tables() { DBUG_ENTER("THD::has_temporary_tables"); bool result= (rgi_slave From 7af133cc111c0fa8f5afa99055d2a22eaac6d94e Mon Sep 17 00:00:00 2001 From: hongdongjian Date: Fri, 25 Mar 2022 19:47:40 +0800 Subject: [PATCH 12/13] MDEV-28177: server_audit; Update the offset of dbName on the aarch64 platform. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the aarch64 platform, MySQL 5.7.33 cannot install this version of the audit plugin, but X86_64 can run well。 --- plugin/server_audit/server_audit.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 14f5b2f98b4..81ec33973e3 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -2328,6 +2328,9 @@ int get_db_mysql57(MYSQL_THD thd, char **name, int *len) #ifdef __x86_64__ db_off= 608; db_len_off= 616; +#elif __aarch64__ + db_off= 632; + db_len_off= 640; #else db_off= 0; db_len_off= 0; @@ -2338,6 +2341,9 @@ int get_db_mysql57(MYSQL_THD thd, char **name, int *len) #ifdef __x86_64__ db_off= 536; db_len_off= 544; +#elif __aarch64__ + db_off= 552; + db_len_off= 560; #else db_off= 0; db_len_off= 0; From 303448bc912486f4766129cc407a5077a3ca4359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 28 Mar 2022 13:36:36 +0300 Subject: [PATCH 13/13] MDEV-27931: buf_page_is_corrupted() wrongly claims corruption In commit 437da7bc54daa131b46900128ebe3ad2ca25c11a (MDEV-19534), the default value of the global variable srv_checksum_algorithm in innochecksum was changed from SRV_CHECKSUM_ALGORITHM_INNODB to implied 0 (innodb_checksum_algorithm=crc32). As a result, the function buf_page_is_corrupted() would by default invoke buf_calc_page_crc32() in innochecksum, and crc32_inited would hold. This would cause "innochecksum" to fail on a particular page. The actual problem is older, introduced in 2011 in mysql/mysql-server@17e497bdb793bc6b8360aa1c626dcd8bb5cfad1b (MySQL 5.6.3). It should affect the validation of pages of old data files that were written with innodb_checksum_algorithm=innodb. When using innodb_checksum_algorithm=crc32 (the default setting since MariaDB Server 10.2), some valid pages would be rejected only because exactly one of the two checksum fields accidentally matches the innodb_checksum_algorithm=crc32 value. buf_page_is_corrupted(): Simplify the logic of non-strict checksum validation, by always invoking buf_calc_page_crc32(). Remove a bogus condition that if only one of the checksum fields contains the value returned by buf_calc_page_crc32(), the page is corrupted. --- storage/innobase/buf/buf0buf.cc | 94 ++++++++------------------------- 1 file changed, 23 insertions(+), 71 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 505539f0217..679f12ca6d1 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -977,8 +977,6 @@ buf_page_is_corrupted( #endif size_t checksum_field1 = 0; size_t checksum_field2 = 0; - uint32_t crc32 = 0; - bool crc32_inited = false; ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE); @@ -1104,8 +1102,13 @@ buf_page_is_corrupted( case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: return !buf_page_is_checksum_valid_none( read_buf, checksum_field1, checksum_field2); + case SRV_CHECKSUM_ALGORITHM_NONE: + /* should have returned false earlier */ + break; case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_INNODB: + const uint32_t crc32 = buf_calc_page_crc32(read_buf); + if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { #ifdef UNIV_INNOCHECKSUM @@ -1121,7 +1124,7 @@ buf_page_is_corrupted( " crc32 = " UINT32PF "; recorded = " ULINTPF ";\n", cur_page_num, buf_calc_page_new_checksum(read_buf), - buf_calc_page_crc32(read_buf), + crc32, checksum_field1); } #endif /* UNIV_INNOCHECKSUM */ @@ -1138,84 +1141,33 @@ buf_page_is_corrupted( != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != BUF_NO_CHECKSUM_MAGIC) { - if (curr_algo == SRV_CHECKSUM_ALGORITHM_CRC32) { - DBUG_EXECUTE_IF( - "page_intermittent_checksum_mismatch", { - static int page_counter; - if (page_counter++ == 2) { - checksum_field2++; - } - }); + DBUG_EXECUTE_IF( + "page_intermittent_checksum_mismatch", { + static int page_counter; + if (page_counter++ == 2) return true; + }); - crc32 = buf_page_check_crc32(read_buf, - checksum_field2); - crc32_inited = true; - - if (checksum_field2 != crc32 - && checksum_field2 - != buf_calc_page_old_checksum(read_buf)) { - return true; - } - } else { - ut_ad(curr_algo - == SRV_CHECKSUM_ALGORITHM_INNODB); - - if (checksum_field2 - != buf_calc_page_old_checksum(read_buf)) { - crc32 = buf_page_check_crc32( - read_buf, checksum_field2); - crc32_inited = true; - - if (checksum_field2 != crc32) { - return true; - } - } + if ((checksum_field1 != crc32 + || checksum_field2 != crc32) + && checksum_field2 + != buf_calc_page_old_checksum(read_buf)) { + return true; } } - if (checksum_field1 == 0 - || checksum_field1 == BUF_NO_CHECKSUM_MAGIC) { - } else if (curr_algo == SRV_CHECKSUM_ALGORITHM_CRC32) { - if (!crc32_inited) { - crc32 = buf_page_check_crc32( - read_buf, checksum_field2); - crc32_inited = true; - } - - if (checksum_field1 != crc32 + switch (checksum_field1) { + case 0: + case BUF_NO_CHECKSUM_MAGIC: + break; + default: + if ((checksum_field1 != crc32 + || checksum_field2 != crc32) && checksum_field1 != buf_calc_page_new_checksum(read_buf)) { return true; } - } else { - ut_ad(curr_algo == SRV_CHECKSUM_ALGORITHM_INNODB); - - if (checksum_field1 - != buf_calc_page_new_checksum(read_buf)) { - - if (!crc32_inited) { - crc32 = buf_page_check_crc32( - read_buf, checksum_field2); - crc32_inited = true; - } - - if (checksum_field1 != crc32) { - return true; - } - } } - if (crc32_inited - && ((checksum_field1 == crc32 - && checksum_field2 != crc32) - || (checksum_field1 != crc32 - && checksum_field2 == crc32))) { - return true; - } - - break; - case SRV_CHECKSUM_ALGORITHM_NONE: - /* should have returned false earlier */ break; }