From dd77f072f9338f784d052ae6f46a04c55eabe3fd Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 10 Jun 2020 08:12:06 +0200 Subject: [PATCH 1/6] MDEV-22841 ut_new_get_key_by_file is unnecessarily expensive Change how lookup for the "auto" PSI_memory_keys is done. Lookup for filename hashes (integers), instead of C strings Generate these hashes at the compile time with constexpr, rather than at runtime. --- storage/innobase/include/ut0new.h | 109 +++++---- storage/innobase/ut/ut0new.cc | 373 +++++++++++++++++++----------- 2 files changed, 302 insertions(+), 180 deletions(-) diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index c35808a56e2..354e6a485f7 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -178,13 +178,13 @@ ut_new_boot(); #ifdef UNIV_PFS_MEMORY -/** Retrieve a memory key (registered with PFS), given a portion of the file -name of the caller. -@param[in] file portion of the filename - basename without an extension -@return registered memory key or PSI_NOT_INSTRUMENTED if not found */ -PSI_memory_key -ut_new_get_key_by_file( - const char* file); +/** +Retrieve a memory key (registered with PFS), +given filename hash of the caller + +@param[in] filename_hash - FILENAME_HASH value of the caller +@return registered memory key or PSI_NOT_INSTRUMENTED */ +PSI_memory_key ut_new_get_key_by_file(uint32_t filename_hash); #endif /* UNIV_PFS_MEMORY */ @@ -293,7 +293,7 @@ public: ) { #ifdef UNIV_PFS_MEMORY - const PSI_memory_key other_key = other.get_mem_key(NULL); + const PSI_memory_key other_key = other.get_mem_key(0); m_key = (other_key != mem_key_std) ? other_key @@ -315,7 +315,7 @@ public: #endif /* UNIV_PFS_MEMORY */ } - pointer allocate(size_type n) { return allocate(n, NULL, NULL); } + pointer allocate(size_type n) { return allocate(n, NULL, 0); } /** Allocate a chunk of memory that can hold 'n_elements' objects of type 'T' and trace the allocation. @@ -333,9 +333,9 @@ public: allocate( size_type n_elements, const_pointer, - const char* + uint32_t #ifdef UNIV_PFS_MEMORY - file /*!< file name of the caller */ + filename_hash /* filename hash of the caller */ #endif , bool set_to_zero = false, @@ -397,7 +397,7 @@ public: #ifdef UNIV_PFS_MEMORY ut_new_pfx_t* pfx = static_cast(ptr); - allocate_trace(total_bytes, file, pfx); + allocate_trace(total_bytes, filename_hash, pfx); return(reinterpret_cast(pfx + 1)); #else @@ -479,7 +479,7 @@ public: reallocate( void* ptr, size_type n_elements, - const char* file) + uint32_t filename_hash) { if (n_elements == 0) { deallocate(static_cast(ptr)); @@ -487,7 +487,7 @@ public: } if (ptr == NULL) { - return(allocate(n_elements, NULL, file, false, false)); + return(allocate(n_elements, NULL, filename_hash, false, false)); } if (n_elements > max_size()) { @@ -530,7 +530,7 @@ public: deallocate_trace(pfx_new); /* pfx_new is set here to describe the new block. */ - allocate_trace(total_bytes, file, pfx_new); + allocate_trace(total_bytes, filename_hash, pfx_new); return(reinterpret_cast(pfx_new + 1)); } @@ -546,9 +546,10 @@ public: pointer new_array( size_type n_elements, - const char* file) + uint32_t filename_hash + ) { - T* p = allocate(n_elements, NULL, file, false, false); + T* p = allocate(n_elements, NULL, filename_hash, false, false); if (p == NULL) { return(NULL); @@ -634,7 +635,7 @@ public: if (pfx != NULL) { #ifdef UNIV_PFS_MEMORY - allocate_trace(n_bytes, NULL, pfx); + allocate_trace(n_bytes, 0, pfx); #endif /* UNIV_PFS_MEMORY */ pfx->m_size = n_bytes; } @@ -687,25 +688,16 @@ public: @return performance schema key */ PSI_memory_key get_mem_key( - const char* file) const + uint32_t filename_hash) const { if (m_key != PSI_NOT_INSTRUMENTED) { return(m_key); } - if (file == NULL) { + if (filename_hash == 0) { return(mem_key_std); } - - /* e.g. "btr0cur", derived from "/path/to/btr0cur.cc" */ - char keyname[FILENAME_MAX]; - const size_t len = ut_basename_noext(file, keyname, - sizeof(keyname)); - /* If sizeof(keyname) was not enough then the output would - be truncated, assert that this did not happen. */ - ut_a(len < sizeof(keyname)); - - const PSI_memory_key key = ut_new_get_key_by_file(keyname); + const PSI_memory_key key = ut_new_get_key_by_file(filename_hash); if (key != PSI_NOT_INSTRUMENTED) { return(key); @@ -747,16 +739,16 @@ private: corresponds to "file", that will be used (see ut_new_boot()) 4. Otherwise, the name associated with mem_key_other will be used. @param[in] size number of bytes that were allocated - @param[in] file file name of the caller or NULL if unknown + @param[in] filename_hash FILENAME_HASH of the caller @param[out] pfx placeholder to store the info which will be needed when freeing the memory */ void allocate_trace( size_t size, - const char* file, + const uint32_t filename_hash, ut_new_pfx_t* pfx) { - const PSI_memory_key key = get_mem_key(file); + const PSI_memory_key key = get_mem_key(filename_hash); pfx->m_key = PSI_MEMORY_CALL(memory_alloc)(key, size, & pfx->m_owner); pfx->m_size = size; @@ -806,6 +798,41 @@ operator!=( #ifdef UNIV_PFS_MEMORY +/* + constexpr trickery ahead. + + Retrieve the FILENAME_HASH = djb2(basename_noext(__FILE__)) at the compile time. + We use the number rather than __FILE__ because integers is better to deal with + (hashing, searching) that C style strings. +*/ + +static constexpr const char * basename_helper(const char* s, const char * last_slash) +{ + return + *s == '\0' ? last_slash : + *s == '/' || *s == '\\' ? basename_helper(s + 1, s + 1) : + basename_helper(s + 1, last_slash); +} + +static constexpr const char* ut_basename(const char *filename) +{ + return basename_helper(filename, filename); +} + +/** Compute djb2 hash for a string. Stop at '.' , or '\0' */ +constexpr uint32_t ut_filename_hash(const char* s, uint32_t h = 5381) +{ + return *s == 0 || *s == '.' ? h : + ut_filename_hash(s + 1, 33 * h + (uint8_t)*s); +} + +/* Force constexpr to be evaluated at compile time.*/ +#define FORCE_CONSTEXPR(expr)[&]() \ +{ static constexpr auto x = (expr); return x; }() + +#define FILENAME_HASH FORCE_CONSTEXPR(ut_filename_hash(ut_basename(__FILE__))) + + /** Allocate, trace the allocation and construct an object. Use this macro instead of 'new' within InnoDB. For example: instead of @@ -823,7 +850,7 @@ pointer must be passed to UT_DELETE() when no longer needed. object if the passed in pointer is NULL, e.g. if allocate() has failed to allocate memory and has returned NULL. */ \ ::new(ut_allocator(key).allocate( \ - sizeof expr, NULL, __FILE__, false, false)) expr + sizeof expr, NULL, FILENAME_HASH, false, false)) expr /** Allocate, trace the allocation and construct an object. Use this macro instead of 'new' within InnoDB and instead of UT_NEW() @@ -871,7 +898,7 @@ The returned pointer must be passed to UT_DELETE_ARRAY(). @param[in] key performance schema memory tracing key @return pointer to the first allocated object or NULL */ #define UT_NEW_ARRAY(type, n_elements, key) \ - ut_allocator(key).new_array(n_elements, __FILE__) + ut_allocator(key).new_array(n_elements, FILENAME_HASH) /** Allocate and account 'n_elements' objects of type 'type'. Use this macro to allocate memory within InnoDB instead of 'new[]' and @@ -902,7 +929,7 @@ ut_delete_array( #define ut_malloc(n_bytes, key) static_cast( \ ut_allocator(key).allocate( \ - n_bytes, NULL, __FILE__, false, false)) + n_bytes, NULL, FILENAME_HASH, false, false)) #define ut_malloc_dontdump(n_bytes, key) static_cast( \ ut_allocator(key).allocate_large( \ @@ -910,23 +937,23 @@ ut_delete_array( #define ut_zalloc(n_bytes, key) static_cast( \ ut_allocator(key).allocate( \ - n_bytes, NULL, __FILE__, true, false)) + n_bytes, NULL, FILENAME_HASH, true, false)) #define ut_malloc_nokey(n_bytes) static_cast( \ ut_allocator(PSI_NOT_INSTRUMENTED).allocate( \ - n_bytes, NULL, __FILE__, false, false)) + n_bytes, NULL, FILENAME_HASH, false, false)) #define ut_zalloc_nokey(n_bytes) static_cast( \ ut_allocator(PSI_NOT_INSTRUMENTED).allocate( \ - n_bytes, NULL, __FILE__, true, false)) + n_bytes, NULL, FILENAME_HASH, true, false)) #define ut_zalloc_nokey_nofatal(n_bytes) static_cast( \ ut_allocator(PSI_NOT_INSTRUMENTED).allocate( \ - n_bytes, NULL, __FILE__, true, false)) + n_bytes, NULL, FILENAME_HASH, true, false)) #define ut_realloc(ptr, n_bytes) static_cast( \ ut_allocator(PSI_NOT_INSTRUMENTED).reallocate( \ - ptr, n_bytes, __FILE__)) + ptr, n_bytes, FILENAME_HASH)) #define ut_free(ptr) ut_allocator(PSI_NOT_INSTRUMENTED).deallocate( \ reinterpret_cast(ptr)) diff --git a/storage/innobase/ut/ut0new.cc b/storage/innobase/ut/ut0new.cc index d1d9e8452f1..057fe86bb08 100644 --- a/storage/innobase/ut/ut0new.cc +++ b/storage/innobase/ut/ut0new.cc @@ -25,7 +25,7 @@ Created May 26, 2014 Vasil Dimov *******************************************************/ #include "univ.i" - +#include /** Maximum number of retries to allocate memory. */ const size_t alloc_max_retries = 60; @@ -44,6 +44,175 @@ PSI_memory_key mem_key_row_merge_sort; PSI_memory_key mem_key_std; #ifdef UNIV_PFS_MEMORY +static const char* auto_event_names[] = +{ + "btr0btr", + "btr0bulk", + "btr0cur", + "btr0defragment", + "btr0pcur", + "btr0sea", + "btr0types", + "buf0buddy", + "buf0buf", + "buf0checksum", + "buf0dblwr", + "buf0dump", + "buf0flu", + "buf0lru", + "buf0rea", + "buf0types", + "data0data", + "data0type", + "data0types", + "db0err", + "dict0boot", + "dict0crea", + "dict0defrag_bg", + "dict0dict", + "dict0load", + "dict0mem", + "dict0pagecompress", + "dict0priv", + "dict0stats", + "dict0stats_bg", + "dict0types", + "dyn0buf", + "dyn0types", + "eval0eval", + "eval0proc", + "fil0crypt", + "fil0fil", + "fil0pagecompress", + "fsp0file", + "fsp0fsp", + "fsp0space", + "fsp0sysspace", + "fsp0types", + "fts0ast", + "fts0blex", + "fts0config", + "fts0fts", + "fts0opt", + "fts0pars", + "fts0plugin", + "fts0priv", + "fts0que", + "fts0sql", + "fts0tlex", + "fts0tokenize", + "fts0types", + "fts0vlc", + "fut0fut", + "fut0lst", + "gis0geo", + "gis0rtree", + "gis0sea", + "gis0type", + "ha0ha", + "ha0storage", + "ha_innodb", + "ha_prototypes", + "handler0alter", + "hash0hash", + "i_s", + "ib0mutex", + "ibuf0ibuf", + "ibuf0types", + "lexyy", + "lock0iter", + "lock0lock", + "lock0prdt", + "lock0priv", + "lock0types", + "lock0wait", + "log0crypt", + "log0log", + "log0recv", + "log0sync", + "log0types", + "mach0data", + "mem0mem", + "mtr0log", + "mtr0mtr", + "mtr0types", + "os0api", + "os0event", + "os0file", + "os0proc", + "os0thread", + "page0cur", + "page0page", + "page0types", + "page0zip", + "pars0grm", + "pars0lex", + "pars0opt", + "pars0pars", + "pars0sym", + "pars0types", + "que0que", + "que0types", + "read0read", + "read0types", + "rem0cmp", + "rem0rec", + "rem0types", + "row0ext", + "row0ftsort", + "row0import", + "row0ins", + "row0log", + "row0merge", + "row0mysql", + "row0purge", + "row0quiesce", + "row0row", + "row0sel", + "row0types", + "row0uins", + "row0umod", + "row0undo", + "row0upd", + "row0vers", + "srv0conc", + "srv0mon", + "srv0srv", + "srv0start", + "sync0arr", + "sync0debug", + "sync0policy", + "sync0rw", + "sync0sync", + "sync0types", + "trx0i_s", + "trx0purge", + "trx0rec", + "trx0roll", + "trx0rseg", + "trx0sys", + "trx0trx", + "trx0types", + "trx0undo", + "trx0xa", + "ut0byte", + "ut0counter", + "ut0crc32", + "ut0dbg", + "ut0list", + "ut0lst", + "ut0mem", + "ut0mutex", + "ut0new", + "ut0pool", + "ut0rbt", + "ut0rnd", + "ut0sort", + "ut0stage", + "ut0ut", + "ut0vec", + "ut0wqueue" +}; /** Auxiliary array of performance schema 'PSI_memory_info'. Each allocation appears in @@ -59,163 +228,89 @@ the list below: Keep this list alphabetically sorted. */ static PSI_memory_info pfs_info[] = { #ifdef BTR_CUR_HASH_ADAPT - {&mem_key_ahi, "adaptive hash index", 0}, + {&mem_key_ahi, "adaptive hash index", 0}, #endif /* BTR_CUR_HASH_ADAPT */ - {&mem_key_buf_buf_pool, "buf_buf_pool", 0}, - {&mem_key_dict_stats_bg_recalc_pool_t, "dict_stats_bg_recalc_pool_t", 0}, - {&mem_key_dict_stats_index_map_t, "dict_stats_index_map_t", 0}, - {&mem_key_dict_stats_n_diff_on_level, "dict_stats_n_diff_on_level", 0}, - {&mem_key_other, "other", 0}, - {&mem_key_row_log_buf, "row_log_buf", 0}, - {&mem_key_row_merge_sort, "row_merge_sort", 0}, - {&mem_key_std, "std", 0}, + {&mem_key_buf_buf_pool, "buf_buf_pool", 0}, + {&mem_key_dict_stats_bg_recalc_pool_t, "dict_stats_bg_recalc_pool_t", 0}, + {&mem_key_dict_stats_index_map_t, "dict_stats_index_map_t", 0}, + {&mem_key_dict_stats_n_diff_on_level, "dict_stats_n_diff_on_level", 0}, + {&mem_key_other, "other", 0}, + {&mem_key_row_log_buf, "row_log_buf", 0}, + {&mem_key_row_merge_sort, "row_merge_sort", 0}, + {&mem_key_std, "std", 0}, }; -/** Map used for default performance schema keys, based on file name of the -caller. The key is the file name of the caller and the value is a pointer -to a PSI_memory_key variable to be passed to performance schema methods. -We use ut_strcmp_functor because by default std::map will compare the pointers -themselves (cont char*) and not do strcmp(). */ -typedef std::map - mem_keys_auto_t; - -/** Map of filename/pfskey, used for tracing allocations that have not -provided a manually created pfs key. This map is only ever modified (bulk -insert) at startup in a single-threaded environment by ut_new_boot(). -Later it is only read (only std::map::find() is called) from multithreaded -environment, thus it is not protected by any latch. */ -static mem_keys_auto_t mem_keys_auto; - -#endif /* UNIV_PFS_MEMORY */ +static const int NKEYS = static_castUT_ARR_SIZE(auto_event_names); +std::pair search_array[NKEYS]; /** Setup the internal objects needed for UT_NEW() to operate. This must be called before the first call to UT_NEW(). */ -void -ut_new_boot() +void ut_new_boot() { -#ifdef UNIV_PFS_MEMORY - static const char* auto_event_names[] = { - /* Keep this list alphabetically sorted. */ - "btr0btr", - "btr0bulk", - "btr0cur", - "btr0pcur", - "btr0sea", - "buf0buf", - "buf0dblwr", - "buf0dump", - "buf0flu", - "buf0lru", - "dict0dict", - "dict0mem", - "dict0stats", - "dict0stats_bg", - "eval0eval", - "fil0fil", - "fsp0file", - "fsp0space", - "fsp0sysspace", - "fts0ast", - "fts0config", - "fts0fts", - "fts0opt", - "fts0pars", - "fts0que", - "fts0sql", - "gis0sea", - "ha0ha", - "ha_innodb", - "handler0alter", - "hash0hash", - "i_s", - "ibuf0ibuf", - "lexyy", - "lock0lock", - "log0log", - "log0recv", - "mem0mem", - "os0event", - "os0file", - "page0cur", - "page0zip", - "pars0lex", - "read0read", - "rem0rec", - "row0ftsort", - "row0import", - "row0log", - "row0merge", - "row0mysql", - "row0sel", - "srv0conc", - "srv0srv", - "srv0start", - "sync0arr", - "sync0debug", - "sync0rw", - "sync0types", - "trx0i_s", - "trx0purge", - "trx0roll", - "trx0rseg", - "trx0sys", - "trx0trx", - "trx0undo", - "ut0list", - "ut0mem", - "ut0mutex", - "ut0pool", - "ut0rbt", - "ut0wqueue", - }; - static const size_t n_auto = UT_ARR_SIZE(auto_event_names); - static PSI_memory_key auto_event_keys[n_auto]; - static PSI_memory_info pfs_info_auto[n_auto]; + PSI_MEMORY_CALL(register_memory)("innodb", pfs_info, UT_ARR_SIZE(pfs_info)); - for (size_t i = 0; i < n_auto; i++) { + static PSI_memory_key auto_event_keys[NKEYS]; + static PSI_memory_info pfs_info_auto[NKEYS]; + for (int i= 0; i < NKEYS; i++) + { + pfs_info_auto[i]= {&auto_event_keys[i], auto_event_names[i], 0}; + } - const std::pair ret - MY_ATTRIBUTE((unused)) - = mem_keys_auto.insert( - mem_keys_auto_t::value_type(auto_event_names[i], - &auto_event_keys[i])); + PSI_MEMORY_CALL(register_memory)("innodb", pfs_info_auto,NKEYS); - /* ret.second is true if new element has been inserted */ - ut_a(ret.second); + if (auto_event_keys[0] == PSI_NOT_INSTRUMENTED) + return; // PSI is off - /* e.g. "btr0btr" */ - pfs_info_auto[i].m_name = auto_event_names[i]; + for (int i= 0; i < NKEYS; i++) + { + search_array[i]= {ut_filename_hash(auto_event_names[i]), auto_event_keys[i]}; + } - /* a pointer to the pfs key */ - pfs_info_auto[i].m_key = &auto_event_keys[i]; + std::sort(search_array, std::end(search_array)); - pfs_info_auto[i].m_flags = 0; - } +#ifdef UNIV_DEBUG + /* assumption that hash value is not 0 in ut0new.h, get_mem_key() */ + ut_ad(search_array[0].first); - PSI_MEMORY_CALL(register_memory)("innodb", pfs_info, static_cast( - UT_ARR_SIZE(pfs_info))); - PSI_MEMORY_CALL(register_memory)("innodb", pfs_info_auto, - static_cast(n_auto)); -#endif /* UNIV_PFS_MEMORY */ + /* Check for hash duplicates */ + for(int i= 0; i < NKEYS-1; i++) + { + if (search_array[i].first == search_array[i + 1].first) + { + // This can only happen if autoevent_names was updated + // previously, or the hash function changed + ib::fatal() << __FILE__ "Duplicates found in filename hashes"; + } + } +#endif } -#ifdef UNIV_PFS_MEMORY +/** Retrieve a memory key (registered with PFS), corresponding to source file hash. -/** Retrieve a memory key (registered with PFS), given a portion of the file -name of the caller. -@param[in] file portion of the filename - basename without an extension -@return registered memory key or PSI_NOT_INSTRUMENTED if not found */ -PSI_memory_key -ut_new_get_key_by_file( - const char* file) +@param[in] filename_hash - hash value (computed at compile time) of a ut_filename_hash + for a one of the auto_event_names. +@return registered memory key or PSI_NOT_INSTRUMENTED +*/ +PSI_memory_key ut_new_get_key_by_file(uint32_t filename_hash) { - mem_keys_auto_t::const_iterator el = mem_keys_auto.find(file); + if(search_array[0].second == PSI_NOT_INSTRUMENTED) + { + // PSI is off. + return PSI_NOT_INSTRUMENTED; + } - if (el != mem_keys_auto.end()) { - return(*(el->second)); - } + std::pair e{ filename_hash, 0 }; + auto result= std::lower_bound(search_array, std::end(search_array), e); + if (result != std::end(search_array) && result->first == filename_hash) + return result->second; - return(PSI_NOT_INSTRUMENTED); +#ifdef UNIV_DEBUG + ib::fatal() << __FILE__ " ut_new_get_key_by_file : hash not found"; +#endif + + return PSI_NOT_INSTRUMENTED; } -#endif /* UNIV_PFS_MEMORY */ +#else /* UNIV_PFS_MEMORY */ +void ut_new_boot(){} +#endif From dc068734743490d41c58df9bef897eb3b89c7472 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 10 Jun 2020 18:41:59 +1000 Subject: [PATCH 2/6] cmake: merge_static_libs - correct duplicate assumptions (#1583) This corrects build failures on ppc64{,le} with the WITH_EMBEDDED_SERVER option enabled. MDEV-22641 added an unusual case in which the same object file in was included twice with a different function defination. The original cmake/merge_archives_unix.cmake did not tolerate such eventualities. So we move to the highest voted answer on Stack Overflow for the merging of static libraries. https://stackoverflow.com/questions/3821916/how-to-merge-two-ar-static-libraries-into-one Thin archives generated compile failures and the libtool mechanism would of been another dependency and using .la files that isn't part of a normal cmake output. The straight Apple mechanism of libtool with static archives also failed on Linux. This leaves the MRI script mechansim which was implemented in this change. --- .gitignore | 2 ++ cmake/libutils.cmake | 32 ++++++++++++++++++------- cmake/merge_archives_unix.cmake | 42 +++------------------------------ 3 files changed, 29 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 83ba4b048ad..e38b56e5894 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ .*.swp *.ninja .ninja_* +*.mri +*.mri.tpl .gdb_history .vs/ errmsg.sys diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake index c7b01bf8429..25fef77b472 100644 --- a/cmake/libutils.cmake +++ b/cmake/libutils.cmake @@ -127,7 +127,8 @@ ENDMACRO() # Merge static libraries into a big static lib. The resulting library # should not not have dependencies on other static libraries. -# We use it in MySQL to merge mysys,dbug,vio etc into mysqlclient +# We use it in MariaDB to merge mysys,dbug,vio etc into the embedded server +# mariadbd. MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE) # To produce a library we need at least one source file. @@ -196,18 +197,33 @@ MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE) ) ELSE() # Generic Unix, Cygwin or MinGW. In post-build step, call - # script, that extracts objects from archives with "ar x" - # and repacks them with "ar r" + # script, that uses a MRI script to append static archives. + IF(CMAKE_VERSION VERSION_LESS "3.0") + SET(MRI_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.mri") + ELSE() + SET(MRI_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-$.mri") + ENDIF() + SET(MRI_SCRIPT_TPL "${MRI_SCRIPT}.tpl") + + SET(SCRIPT_CONTENTS "CREATE $\n") + FOREACH(LIB ${STATIC_LIBS}) + STRING(APPEND SCRIPT_CONTENTS "ADDLIB ${LIB}\n") + ENDFOREACH() + STRING(APPEND SCRIPT_CONTENTS "SAVE\nEND\n") + FILE(WRITE ${MRI_SCRIPT_TPL} "${SCRIPT_CONTENTS}") + FILE(GENERATE OUTPUT ${MRI_SCRIPT} INPUT ${MRI_SCRIPT_TPL}) + ADD_CUSTOM_COMMAND(TARGET ${TARGET} POST_BUILD + DEPENDS ${MRI_SCRIPT} COMMAND ${CMAKE_COMMAND} - -DTARGET_LOCATION="$" - -DTARGET="${TARGET}" - -DSTATIC_LIBS="${STATIC_LIBS}" - -DCMAKE_CURRENT_BINARY_DIR="${CMAKE_CURRENT_BINARY_DIR}" + ARGS + -DTARGET_SCRIPT="${MRI_SCRIPT}" -DCMAKE_AR="${CMAKE_AR}" - -DCMAKE_RANLIB="${CMAKE_RANLIB}" -P "${MYSQL_CMAKE_SCRIPT_DIR}/merge_archives_unix.cmake" + COMMAND ${CMAKE_RANLIB} + ARGS $ ) + SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${MRI_SCRIPT_TPL}) ENDIF() ENDIF() ENDMACRO() diff --git a/cmake/merge_archives_unix.cmake b/cmake/merge_archives_unix.cmake index 09f4fbf2506..8e5086bb5b7 100644 --- a/cmake/merge_archives_unix.cmake +++ b/cmake/merge_archives_unix.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2009 Sun Microsystems, Inc. +# Copyright (c) 2020 IBM # Use is subject to license terms. # # This program is free software; you can redistribute it and/or modify @@ -14,43 +14,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -FILE(REMOVE "${TARGET_LOCATION}") - -SET(TEMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/merge_archives_${TARGET}) -MAKE_DIRECTORY(${TEMP_DIR}) -# Extract each archive to its own subdirectory(avoid object filename clashes) -SEPARATE_ARGUMENTS(STATIC_LIBS UNIX_COMMAND "${STATIC_LIBS}") -FOREACH(LIB ${STATIC_LIBS}) - GET_FILENAME_COMPONENT(NAME_NO_EXT ${LIB} NAME_WE) - SET(TEMP_SUBDIR ${TEMP_DIR}/${NAME_NO_EXT}) - MAKE_DIRECTORY(${TEMP_SUBDIR}) - EXECUTE_PROCESS( - COMMAND ${CMAKE_AR} -x ${LIB} - WORKING_DIRECTORY ${TEMP_SUBDIR} - ) - - FILE(GLOB_RECURSE LIB_OBJECTS "${TEMP_SUBDIR}/*") - SET(OBJECTS ${OBJECTS} ${LIB_OBJECTS}) -ENDFOREACH() - -# Use relative paths, makes command line shorter. -GET_FILENAME_COMPONENT(ABS_TEMP_DIR ${TEMP_DIR} ABSOLUTE) -FOREACH(OBJ ${OBJECTS}) - FILE(RELATIVE_PATH OBJ ${ABS_TEMP_DIR} ${OBJ}) - FILE(TO_NATIVE_PATH ${OBJ} OBJ) - SET(ALL_OBJECTS ${ALL_OBJECTS} ${OBJ}) -ENDFOREACH() - -FILE(TO_NATIVE_PATH ${TARGET_LOCATION} ${TARGET_LOCATION}) -# Now pack the objects into library with ar. EXECUTE_PROCESS( - COMMAND ${CMAKE_AR} -r ${TARGET_LOCATION} ${ALL_OBJECTS} - WORKING_DIRECTORY ${TEMP_DIR} + COMMAND ${CMAKE_AR} -M + INPUT_FILE ${TARGET_SCRIPT} ) -EXECUTE_PROCESS( - COMMAND ${CMAKE_RANLIB} ${TARGET_LOCATION} - WORKING_DIRECTORY ${TEMP_DIR} -) - -# Cleanup -FILE(REMOVE_RECURSE ${TEMP_DIR}) From 800eee42cea177849a1bd77d37d6121bc79d57ea Mon Sep 17 00:00:00 2001 From: Sujatha Date: Tue, 9 Jun 2020 14:36:29 +0530 Subject: [PATCH 3/6] MDEV-22059: MSAN report at replicate_ignore_table_grant Analysis: ======== List of values provided for "replicate_ignore_table" and "replicate_do_table" are stored in HASH. When an empty list is provided the HASH structure doesn't get initialized. Existing code treats empty element list as an error and tries to clean the uninitialized HASH. This results in above MSAN issue. Fix: === The clean up should be initiated only when there is an error while parsing the 'replicate_do_table' or 'replicate_ignore_table' list and the HASH is in initialized state. Otherwise for empty list it should simply return success. --- sql/rpl_filter.cc | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/sql/rpl_filter.cc b/sql/rpl_filter.cc index 24b76cf7c42..5c4a4d9f58a 100644 --- a/sql/rpl_filter.cc +++ b/sql/rpl_filter.cc @@ -350,16 +350,22 @@ Rpl_filter::set_do_table(const char* table_spec) int status; if (do_table_inited) - my_hash_reset(&do_table); - - status= parse_filter_rule(table_spec, &Rpl_filter::add_do_table); - - if (!do_table.records) { my_hash_free(&do_table); do_table_inited= 0; } + status= parse_filter_rule(table_spec, &Rpl_filter::add_do_table); + + if (do_table_inited && status) + { + if (!do_table.records) + { + my_hash_free(&do_table); + do_table_inited= 0; + } + } + return status; } @@ -370,16 +376,22 @@ Rpl_filter::set_ignore_table(const char* table_spec) int status; if (ignore_table_inited) - my_hash_reset(&ignore_table); - - status= parse_filter_rule(table_spec, &Rpl_filter::add_ignore_table); - - if (!ignore_table.records) { my_hash_free(&ignore_table); ignore_table_inited= 0; } + status= parse_filter_rule(table_spec, &Rpl_filter::add_ignore_table); + + if (ignore_table_inited && status) + { + if (!ignore_table.records) + { + my_hash_free(&ignore_table); + ignore_table_inited= 0; + } + } + return status; } From 680a13957c0fea0a69b35e54922df1bce0c5226d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Sun, 17 May 2020 12:38:51 +0300 Subject: [PATCH 4/6] MDEV-19933: Sync mariadb-common and update-alternatives based /etc/mysql/ Split the big my.cnf into multiple smaller files with the same filenames and contents as official Debian/Ubuntu packaging has. The config contents stays the same apart from following additions which the original MariaDB upstream configs had and probably needs to be kept: - lc-messages=en_US and skip-external-locking in server config Configs the original MariaDB upstream had that are seemingly unnecessary and thus removed: - port=3306 removed from the client config - log_warnings=2 removed from server config Also adopt update-alternatives system using mysql-common/configure-symlinks. This way it is aligned with downstream Debian/Ubuntu packaging. --- debian/additions/mariadb.cnf | 34 ++-- debian/additions/mariadb.conf.d/50-client.cnf | 25 +++ .../mariadb.conf.d/50-mysql-clients.cnf | 24 +++ .../mariadb.conf.d/50-mysqld_safe.cnf | 30 +++ debian/additions/mariadb.conf.d/50-server.cnf | 133 ++++++++++++ debian/additions/mariadb.conf.d/60-galera.cnf | 19 ++ debian/additions/my.cnf | 189 +----------------- debian/additions/mysqld_safe_syslog.cnf | 3 - debian/control | 7 +- debian/mariadb-client-10.5.install | 3 + debian/mariadb-common.dirs | 2 +- debian/mariadb-common.postinst | 29 +++ debian/mariadb-common.postrm | 17 +- debian/mariadb-server-10.5.install | 3 +- 14 files changed, 301 insertions(+), 217 deletions(-) create mode 100644 debian/additions/mariadb.conf.d/50-client.cnf create mode 100644 debian/additions/mariadb.conf.d/50-mysql-clients.cnf create mode 100644 debian/additions/mariadb.conf.d/50-mysqld_safe.cnf create mode 100644 debian/additions/mariadb.conf.d/50-server.cnf create mode 100644 debian/additions/mariadb.conf.d/60-galera.cnf mode change 100644 => 120000 debian/additions/my.cnf delete mode 100644 debian/additions/mysqld_safe_syslog.cnf create mode 100644 debian/mariadb-common.postinst diff --git a/debian/additions/mariadb.cnf b/debian/additions/mariadb.cnf index da21212faef..c6fb2f781a2 100644 --- a/debian/additions/mariadb.cnf +++ b/debian/additions/mariadb.cnf @@ -1,19 +1,23 @@ -# MariaDB-specific config file. -# Read by /etc/mysql/my.cnf - -[client] -# Default is Latin1, if you need UTF-8 set this (also in server section) -#default-character-set = utf8 - -[mysqld] +# The MariaDB configuration file # -# * Character sets -# -# Default is Latin1, if you need UTF-8 set all this (also in client section) +# The MariaDB/MySQL tools read configuration files in the following order: +# 1. "/etc/mysql/mariadb.cnf" (this file) to set global defaults, +# 2. "/etc/mysql/conf.d/*.cnf" to set global options. +# 3. "/etc/mysql/mariadb.conf.d/*.cnf" to set MariaDB-only options. +# 4. "~/.my.cnf" to set user-specific options. # -#character-set-server = utf8 -#collation-server = utf8_general_ci -#character_set_server = utf8 -#collation_server = utf8_general_ci +# If the same option is defined multiple times, the last one will apply. +# +# One can use all long options that the program supports. +# Run program with --help to get a list of available options and with +# --print-defaults to see which it would actually understand and use. + +# +# This group is read both by the client and the server +# use it for options that affect everything +# +[client-server] + # Import all .cnf files from configuration directory +!includedir /etc/mysql/conf.d/ !includedir /etc/mysql/mariadb.conf.d/ diff --git a/debian/additions/mariadb.conf.d/50-client.cnf b/debian/additions/mariadb.conf.d/50-client.cnf new file mode 100644 index 00000000000..b509f191047 --- /dev/null +++ b/debian/additions/mariadb.conf.d/50-client.cnf @@ -0,0 +1,25 @@ +# +# This group is read by the client library +# Use it for options that affect all clients, but not the server +# + +[client] +# Default is Latin1, if you need UTF-8 set this (also in server section) +default-character-set = utf8mb4 + +# socket location +socket = /var/run/mysqld/mysqld.sock + +# Example of client certificate usage +# ssl-cert=/etc/mysql/client-cert.pem +# ssl-key=/etc/mysql/client-key.pem +# +# Allow only TLS encrypted connections +# ssl-verify-server-cert=on + +# This group is *never* read by mysql client library, though this +# /etc/mysql/mariadb.cnf.d/client.cnf file is not read by Oracle MySQL +# client anyway. +# If you use the same .cnf file for MySQL and MariaDB, +# use it for MariaDB-only client options +[client-mariadb] diff --git a/debian/additions/mariadb.conf.d/50-mysql-clients.cnf b/debian/additions/mariadb.conf.d/50-mysql-clients.cnf new file mode 100644 index 00000000000..55cfda26158 --- /dev/null +++ b/debian/additions/mariadb.conf.d/50-mysql-clients.cnf @@ -0,0 +1,24 @@ +# +# These groups are read by MariaDB command-line tools +# Use it for options that affect only one utility +# + +[mysql] +# Default is Latin1, if you need UTF-8 set this (also in server section) +default-character-set = utf8mb4 + +[mysql_upgrade] + +[mysqladmin] + +[mysqlbinlog] + +[mysqlcheck] + +[mysqldump] + +[mysqlimport] + +[mysqlshow] + +[mysqlslap] diff --git a/debian/additions/mariadb.conf.d/50-mysqld_safe.cnf b/debian/additions/mariadb.conf.d/50-mysqld_safe.cnf new file mode 100644 index 00000000000..141d51f61a2 --- /dev/null +++ b/debian/additions/mariadb.conf.d/50-mysqld_safe.cnf @@ -0,0 +1,30 @@ +# NOTE: This file is read only by the traditional SysV init script, not systemd. +# MariaDB systemd does _not_ utilize mysqld_safe nor read this file. +# +# For similar behaviour, systemd users should create the following file: +# /etc/systemd/system/mariadb.service.d/migrated-from-my.cnf-settings.conf +# +# To achieve the same result as the default 50-mysqld_safe.cnf, please create +# /etc/systemd/system/mariadb.service.d/migrated-from-my.cnf-settings.conf +# with the following contents: +# +# [Service] +# User=mysql +# StandardOutput=syslog +# StandardError=syslog +# SyslogFacility=daemon +# SyslogLevel=err +# SyslogIdentifier=mysqld +# +# For more information, please read https://mariadb.com/kb/en/mariadb/systemd/ +# + +[mysqld_safe] +# This will be passed to all mysql clients +# It has been reported that passwords should be enclosed with ticks/quotes +# especially if they contain "#" chars... +# Remember to edit /etc/mysql/debian.cnf when changing the socket location. +socket = /var/run/mysqld/mysqld.sock +nice = 0 +skip_log_error +syslog diff --git a/debian/additions/mariadb.conf.d/50-server.cnf b/debian/additions/mariadb.conf.d/50-server.cnf new file mode 100644 index 00000000000..3c39e92bd7d --- /dev/null +++ b/debian/additions/mariadb.conf.d/50-server.cnf @@ -0,0 +1,133 @@ +# +# These groups are read by MariaDB server. +# Use it for options that only the server (but not clients) should see +# +# See the examples of server my.cnf files in /usr/share/mysql + +# this is read by the standalone daemon and embedded servers +[server] + +# this is only for the mysqld standalone daemon +[mysqld] + +# +# * Basic Settings +# +user = mysql +pid-file = /run/mysqld/mysqld.pid +socket = /run/mysqld/mysqld.sock +#port = 3306 +basedir = /usr +datadir = /var/lib/mysql +tmpdir = /tmp +lc-messages-dir = /usr/share/mysql +#skip-external-locking + +# Instead of skip-networking the default is now to listen only on +# localhost which is more compatible and is not less secure. +bind-address = 127.0.0.1 + +# +# * Fine Tuning +# +#key_buffer_size = 16M +#max_allowed_packet = 16M +#thread_stack = 192K +#thread_cache_size = 8 +# This replaces the startup script and checks MyISAM tables if needed +# the first time they are touched +#myisam_recover_options = BACKUP +#max_connections = 100 +#table_cache = 64 +#thread_concurrency = 10 + +# +# * Query Cache Configuration +# +#query_cache_limit = 1M +query_cache_size = 16M + +# +# * Logging and Replication +# +# Both location gets rotated by the cronjob. +# Be aware that this log type is a performance killer. +# As of 5.1 you can enable the log at runtime! +#general_log_file = /var/log/mysql/mysql.log +#general_log = 1 +# +# Error log - should be very few entries. +# +log_error = /var/log/mysql/error.log +# +# Enable the slow query log to see queries with especially long duration +#slow_query_log_file = /var/log/mysql/mariadb-slow.log +#long_query_time = 10 +#log_slow_rate_limit = 1000 +#log_slow_verbosity = query_plan +#log-queries-not-using-indexes +# +# The following can be used as easy to replay backup logs or for replication. +# note: if you are setting up a replication slave, see README.Debian about +# other settings you may need to change. +#server-id = 1 +#log_bin = /var/log/mysql/mysql-bin.log +expire_logs_days = 10 +#max_binlog_size = 100M +#binlog_do_db = include_database_name +#binlog_ignore_db = exclude_database_name + +# +# * Security Features +# +# Read the manual, too, if you want chroot! +#chroot = /var/lib/mysql/ +# +# For generating SSL certificates you can use for example the GUI tool "tinyca". +# +#ssl-ca = /etc/mysql/cacert.pem +#ssl-cert = /etc/mysql/server-cert.pem +#ssl-key = /etc/mysql/server-key.pem +# +# Accept only connections using the latest and most secure TLS protocol version. +# ..when MariaDB is compiled with OpenSSL: +#ssl-cipher = TLSv1.2 +# ..when MariaDB is compiled with YaSSL (default in Debian): +#ssl = on + +# +# * Character sets +# +# MySQL/MariaDB default is Latin1, but in Debian we rather default to the full +# utf8 4-byte character set. See also client.cnf +# +character-set-server = utf8mb4 +collation-server = utf8mb4_general_ci + +# +# * InnoDB +# +# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. +# Read the manual for more InnoDB related options. There are many! + +# +# * Unix socket authentication plugin is built-in since 10.0.22-6 +# +# Needed so the root database user can authenticate without a password but +# only when running as the unix root user. +# +# Also available for other users if required. +# See https://mariadb.com/kb/en/unix_socket-authentication-plugin/ + +# this is only for embedded server +[embedded] + +# This group is only read by MariaDB servers, not by MySQL. +# If you use the same .cnf file for MySQL and MariaDB, +# you can put MariaDB-only options here +[mariadb] + +# This group is only read by MariaDB-10.5 servers. +# If you use the same .cnf file for MariaDB of different versions, +# use this group for options that older servers don't understand +[mariadb-10.5] diff --git a/debian/additions/mariadb.conf.d/60-galera.cnf b/debian/additions/mariadb.conf.d/60-galera.cnf new file mode 100644 index 00000000000..7249a0f1ec8 --- /dev/null +++ b/debian/additions/mariadb.conf.d/60-galera.cnf @@ -0,0 +1,19 @@ +# +# * Galera-related settings +# +[galera] +# Mandatory settings +#wsrep_on=ON +#wsrep_provider= +#wsrep_cluster_address= +#binlog_format=row +#default_storage_engine=InnoDB +#innodb_autoinc_lock_mode=2 +# +# Allow server to accept connections on all interfaces. +# +#bind-address=0.0.0.0 +# +# Optional setting +#wsrep_slave_threads=1 +#innodb_flush_log_at_trx_commit=0 diff --git a/debian/additions/my.cnf b/debian/additions/my.cnf deleted file mode 100644 index 77596bc308b..00000000000 --- a/debian/additions/my.cnf +++ /dev/null @@ -1,188 +0,0 @@ -# MariaDB database server configuration file. -# -# You can copy this file to one of: -# - "/etc/mysql/my.cnf" to set global options, -# - "~/.my.cnf" to set user-specific options. -# -# One can use all long options that the program supports. -# Run program with --help to get a list of available options and with -# --print-defaults to see which it would actually understand and use. -# -# For explanations see -# http://dev.mysql.com/doc/mysql/en/server-system-variables.html - -# This will be passed to all mysql clients -# It has been reported that passwords should be enclosed with ticks/quotes -# escpecially if they contain "#" chars... -# Remember to edit /etc/mysql/debian.cnf when changing the socket location. -[client] -port = 3306 -socket = /var/run/mysqld/mysqld.sock - -# Here is entries for some specific programs -# The following values assume you have at least 32M ram - -# This was formally known as [safe_mysqld]. Both versions are currently parsed. -[mysqld_safe] -socket = /var/run/mysqld/mysqld.sock -nice = 0 - -[mysqld] -# -# * Basic Settings -# -user = mysql -pid-file = /var/run/mysqld/mysqld.pid -socket = /var/run/mysqld/mysqld.sock -port = 3306 -basedir = /usr -datadir = /var/lib/mysql -tmpdir = /tmp -lc_messages_dir = /usr/share/mysql -lc_messages = en_US -skip-external-locking -# -# Instead of skip-networking the default is now to listen only on -# localhost which is more compatible and is not less secure. -bind-address = 127.0.0.1 -# -# * Fine Tuning -# -max_connections = 100 -connect_timeout = 5 -wait_timeout = 600 -max_allowed_packet = 16M -thread_cache_size = 128 -sort_buffer_size = 4M -bulk_insert_buffer_size = 16M -tmp_table_size = 32M -max_heap_table_size = 32M -# -# * MyISAM -# -# This replaces the startup script and checks MyISAM tables if needed -# the first time they are touched. On error, make copy and try a repair. -myisam_recover_options = BACKUP -key_buffer_size = 128M -#open-files-limit = 2000 -table_open_cache = 400 -myisam_sort_buffer_size = 512M -concurrent_insert = 2 -read_buffer_size = 2M -read_rnd_buffer_size = 1M -# -# * Query Cache Configuration -# -# Cache only tiny result sets, so we can fit more in the query cache. -query_cache_limit = 128K -query_cache_size = 64M -# for more write intensive setups, set to DEMAND or OFF -#query_cache_type = DEMAND -# -# * Logging and Replication -# -# Both location gets rotated by the cronjob. -# Be aware that this log type is a performance killer. -# As of 5.1 you can enable the log at runtime! -#general_log_file = /var/log/mysql/mysql.log -#general_log = 1 -# -# Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. -# -# we do want to know about network errors and such -log_warnings = 2 -# -# Enable the slow query log to see queries with especially long duration -#slow_query_log[={0|1}] -slow_query_log_file = /var/log/mysql/mariadb-slow.log -long_query_time = 10 -#log_slow_rate_limit = 1000 -log_slow_verbosity = query_plan - -#log-queries-not-using-indexes -#log_slow_admin_statements -# -# The following can be used as easy to replay backup logs or for replication. -# note: if you are setting up a replication slave, see README.Debian about -# other settings you may need to change. -#server-id = 1 -#report_host = master1 -#auto_increment_increment = 2 -#auto_increment_offset = 1 -log_bin = /var/log/mysql/mariadb-bin -log_bin_index = /var/log/mysql/mariadb-bin.index -# not fab for performance, but safer -#sync_binlog = 1 -expire_logs_days = 10 -max_binlog_size = 100M -# slaves -#relay_log = /var/log/mysql/relay-bin -#relay_log_index = /var/log/mysql/relay-bin.index -#relay_log_info_file = /var/log/mysql/relay-bin.info -#log_slave_updates -#read_only -# -# If applications support it, this stricter sql_mode prevents some -# mistakes like inserting invalid dates etc. -#sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL -# -# * InnoDB -# -# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. -# Read the manual for more InnoDB related options. There are many! -default_storage_engine = InnoDB -innodb_buffer_pool_size = 256M -innodb_log_buffer_size = 8M -innodb_file_per_table = 1 -innodb_open_files = 400 -innodb_io_capacity = 400 -innodb_flush_method = O_DIRECT -# -# * Security Features -# -# Read the manual, too, if you want chroot! -# chroot = /var/lib/mysql/ -# -# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". -# -# ssl-ca=/etc/mysql/cacert.pem -# ssl-cert=/etc/mysql/server-cert.pem -# ssl-key=/etc/mysql/server-key.pem - -# -# * Galera-related settings -# -[galera] -# Mandatory settings -#wsrep_on=ON -#wsrep_provider= -#wsrep_cluster_address= -#binlog_format=row -#default_storage_engine=InnoDB -#innodb_autoinc_lock_mode=2 -# -# Allow server to accept connections on all interfaces. -# -#bind-address=0.0.0.0 -# -# Optional setting -#wsrep_slave_threads=1 -#innodb_flush_log_at_trx_commit=0 - -[mysqldump] -quick -quote-names -max_allowed_packet = 16M - -[mysql] -#no-auto-rehash # faster start of mysql but no tab completion - -[isamchk] -key_buffer = 16M - -# -# * IMPORTANT: Additional settings that can override those from this file! -# The files must end with '.cnf', otherwise they'll be ignored. -# -!include /etc/mysql/mariadb.cnf -!includedir /etc/mysql/conf.d/ diff --git a/debian/additions/my.cnf b/debian/additions/my.cnf new file mode 120000 index 00000000000..d7048c65c43 --- /dev/null +++ b/debian/additions/my.cnf @@ -0,0 +1 @@ +mariadb.cnf \ No newline at end of file diff --git a/debian/additions/mysqld_safe_syslog.cnf b/debian/additions/mysqld_safe_syslog.cnf deleted file mode 100644 index 08984c1716b..00000000000 --- a/debian/additions/mysqld_safe_syslog.cnf +++ /dev/null @@ -1,3 +0,0 @@ -[mysqld_safe] -skip_log_error -syslog diff --git a/debian/control b/debian/control index c0132f07dce..71feb204fbd 100644 --- a/debian/control +++ b/debian/control @@ -230,16 +230,15 @@ Description: MariaDB database common files (e.g. /etc/mysql/my.cnf) Package: mariadb-common Architecture: all -Depends: mysql-common, +Depends: mysql-common (>= 5.6.25), ${misc:Depends} -Description: MariaDB database common files (e.g. /etc/mysql/conf.d/mariadb.cnf) +Description: MariaDB common configuration files MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package includes files needed by all versions of the client library - (e.g. /etc/mysql/conf.d/mariadb.cnf). + This package includes configuration files common to all MariaDB programs. Package: mariadb-client-core-10.5 Architecture: any diff --git a/debian/mariadb-client-10.5.install b/debian/mariadb-client-10.5.install index 35b614cd976..b299d3034c9 100644 --- a/debian/mariadb-client-10.5.install +++ b/debian/mariadb-client-10.5.install @@ -1,5 +1,8 @@ debian/additions/innotop/innotop usr/bin/ debian/additions/mariadb-report usr/bin/ +debian/additions/mariadb.conf.d/50-client.cnf etc/mysql/mariadb.conf.d +debian/additions/mariadb.conf.d/50-mysql-clients.cnf etc/mysql/mariadb.conf.d +debian/additions/mariadb.conf.d/60-galera.cnf etc/mysql/mariadb.conf.d usr/bin/mariadb-access usr/bin/mariadb-admin usr/bin/mariadb-conv diff --git a/debian/mariadb-common.dirs b/debian/mariadb-common.dirs index 87c71c1c2f5..8aba9560977 100644 --- a/debian/mariadb-common.dirs +++ b/debian/mariadb-common.dirs @@ -1 +1 @@ -etc/mysql/mariadb.conf.d/ +etc/mysql/mariadb.conf.d diff --git a/debian/mariadb-common.postinst b/debian/mariadb-common.postinst new file mode 100644 index 00000000000..2a8037e8c48 --- /dev/null +++ b/debian/mariadb-common.postinst @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e + +case "$1" in + configure) + # New packaging paradigm for my.cnf handling among MySQL variants + # Used in Ubuntu since Dec-2014 and in Debian since Jul-2015 + # + # If the new mysql-common package does not provide + # the update-alternatives facility, notify user about manual fall back + if [ -f /usr/share/mysql-common/configure-symlinks ] + then + /usr/share/mysql-common/configure-symlinks install mariadb "/etc/mysql/mariadb.cnf" + else + # As configure can be called many times, don't re-create the symlink + # if it is there already + if [ ! -L /etc/mysql/my.cnf ] + then + echo "Notice: configure-symlinks trigger could not be called." + echo "Please manually create symlinks by running: " + echo " mv -f /etc/mysql/my.cnf /etc/mysql/my.cnf.old" + echo " ln -sf mariadb.cnf /etc/mysql/my.cnf" + fi + fi + ;; +esac + +#DEBHELPER# diff --git a/debian/mariadb-common.postrm b/debian/mariadb-common.postrm index 14007fb9149..d0bfa266b7d 100644 --- a/debian/mariadb-common.postrm +++ b/debian/mariadb-common.postrm @@ -1,9 +1,16 @@ -#!/bin/bash +#!/bin/sh + set -e -if [ "$1" = "purge" ]; then - rmdir /etc/mysql/conf.d 2>/dev/null || true - rmdir /etc/mysql 2>/dev/null || true -fi +case "$1" in + remove|disappear) + # New packaging paradigm for my.cnf handling among MySQL variants + # Used in Ubuntu since Dec-2014 and in Debian since Jul-2015 + if [ -f /usr/share/mysql-common/configure-symlinks ] + then + /usr/share/mysql-common/configure-symlinks remove mariadb "/etc/mysql/mariadb.cnf" + fi + ;; +esac #DEBHELPER# diff --git a/debian/mariadb-server-10.5.install b/debian/mariadb-server-10.5.install index e6c17bd2d10..ca100506753 100644 --- a/debian/mariadb-server-10.5.install +++ b/debian/mariadb-server-10.5.install @@ -1,7 +1,8 @@ debian/additions/debian-start etc/mysql debian/additions/debian-start.inc.sh usr/share/mysql debian/additions/echo_stderr usr/share/mysql -debian/additions/mysqld_safe_syslog.cnf etc/mysql/conf.d +debian/additions/mariadb.conf.d/50-mysqld_safe.cnf etc/mysql/mariadb.conf.d +debian/additions/mariadb.conf.d/50-server.cnf etc/mysql/mariadb.conf.d debian/additions/source_mariadb-10.5.py usr/share/apport/package-hooks etc/apparmor.d/usr.sbin.mysqld etc/security/user_map.conf From aaaf005ce61a543f1a1470b179ccb535698b36ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Mon, 13 Apr 2020 10:00:43 +0300 Subject: [PATCH 5/6] Deb: Clean up default configs for 10.5 era - Recommend max_allowed_packet=1G which is the same as the default client value. - Remove thread_concurrency removed in 10.5. - Remove query cache, not recommended practice anymore. - Remove binlog_*, should not recommend those too easily but rather require the database administrator to read up on those themselves. - Remove chroot setting, not relevant in modern container era. - Show explicitly innodb_buffer_pool_size example as the most likely thing a database administrator should change. - Don't recommend rate limiting in slow log, logging once in a 1000 would not be optimal for the basic case, hence bad example. - Install the example configs in /usr/share/mysql. - Use correct path /run/ instead of /var/run/. --- debian/additions/mariadb.conf.d/50-client.cnf | 2 +- .../mariadb.conf.d/50-mysqld_safe.cnf | 7 +- debian/additions/mariadb.conf.d/50-server.cnf | 75 ++++++++----------- debian/additions/mariadb.conf.d/60-galera.cnf | 9 ++- debian/mariadb-server-10.5.install | 1 + 5 files changed, 40 insertions(+), 54 deletions(-) diff --git a/debian/additions/mariadb.conf.d/50-client.cnf b/debian/additions/mariadb.conf.d/50-client.cnf index b509f191047..0935b52f36e 100644 --- a/debian/additions/mariadb.conf.d/50-client.cnf +++ b/debian/additions/mariadb.conf.d/50-client.cnf @@ -8,7 +8,7 @@ default-character-set = utf8mb4 # socket location -socket = /var/run/mysqld/mysqld.sock +socket = /run/mysqld/mysqld.sock # Example of client certificate usage # ssl-cert=/etc/mysql/client-cert.pem diff --git a/debian/additions/mariadb.conf.d/50-mysqld_safe.cnf b/debian/additions/mariadb.conf.d/50-mysqld_safe.cnf index 141d51f61a2..914fc1203ea 100644 --- a/debian/additions/mariadb.conf.d/50-mysqld_safe.cnf +++ b/debian/additions/mariadb.conf.d/50-mysqld_safe.cnf @@ -1,7 +1,7 @@ # NOTE: This file is read only by the traditional SysV init script, not systemd. # MariaDB systemd does _not_ utilize mysqld_safe nor read this file. # -# For similar behaviour, systemd users should create the following file: +# For similar behavior, systemd users should create the following file: # /etc/systemd/system/mariadb.service.d/migrated-from-my.cnf-settings.conf # # To achieve the same result as the default 50-mysqld_safe.cnf, please create @@ -17,14 +17,13 @@ # SyslogIdentifier=mysqld # # For more information, please read https://mariadb.com/kb/en/mariadb/systemd/ -# [mysqld_safe] # This will be passed to all mysql clients # It has been reported that passwords should be enclosed with ticks/quotes # especially if they contain "#" chars... # Remember to edit /etc/mysql/debian.cnf when changing the socket location. -socket = /var/run/mysqld/mysqld.sock -nice = 0 +socket = /run/mysqld/mysqld.sock +nice = 0 skip_log_error syslog diff --git a/debian/additions/mariadb.conf.d/50-server.cnf b/debian/additions/mariadb.conf.d/50-server.cnf index 3c39e92bd7d..5ac717ca6c6 100644 --- a/debian/additions/mariadb.conf.d/50-server.cnf +++ b/debian/additions/mariadb.conf.d/50-server.cnf @@ -1,8 +1,6 @@ # # These groups are read by MariaDB server. # Use it for options that only the server (but not clients) should see -# -# See the examples of server my.cnf files in /usr/share/mysql # this is read by the standalone daemon and embedded servers [server] @@ -13,6 +11,7 @@ # # * Basic Settings # + user = mysql pid-file = /run/mysqld/mysqld.pid socket = /run/mysqld/mysqld.sock @@ -21,7 +20,12 @@ basedir = /usr datadir = /var/lib/mysql tmpdir = /tmp lc-messages-dir = /usr/share/mysql -#skip-external-locking +lc-messages = en_US +skip-external-locking + +# Broken reverse DNS slows down connections considerably and name resolve is +# safe to skip if there are no "host by domain name" access grants +#skip-name-resolve # Instead of skip-networking the default is now to listen only on # localhost which is more compatible and is not less secure. @@ -30,8 +34,9 @@ bind-address = 127.0.0.1 # # * Fine Tuning # -#key_buffer_size = 16M -#max_allowed_packet = 16M + +#key_buffer_size = 128M +#max_allowed_packet = 1G #thread_stack = 192K #thread_cache_size = 8 # This replaces the startup script and checks MyISAM tables if needed @@ -39,34 +44,29 @@ bind-address = 127.0.0.1 #myisam_recover_options = BACKUP #max_connections = 100 #table_cache = 64 -#thread_concurrency = 10 - -# -# * Query Cache Configuration -# -#query_cache_limit = 1M -query_cache_size = 16M # # * Logging and Replication # + # Both location gets rotated by the cronjob. # Be aware that this log type is a performance killer. -# As of 5.1 you can enable the log at runtime! +# Recommend only changing this at runtime for short testing periods if needed! #general_log_file = /var/log/mysql/mysql.log #general_log = 1 -# -# Error log - should be very few entries. -# -log_error = /var/log/mysql/error.log -# + +# When running under systemd, error logging goes via stdout/stderr to journald +# and when running legacy init error logging goes to syslog due to +# /etc/mysql/conf.d/mariadb.conf.d/50-mysqld_safe.cnf +# Enable this if you want to have error logging into a separate file +#log_error = /var/log/mysql/error.log # Enable the slow query log to see queries with especially long duration #slow_query_log_file = /var/log/mysql/mariadb-slow.log #long_query_time = 10 -#log_slow_rate_limit = 1000 -#log_slow_verbosity = query_plan +#log_slow_verbosity = query_plan,explain #log-queries-not-using-indexes -# +#min_examined_row_limit = 1000 + # The following can be used as easy to replay backup logs or for replication. # note: if you are setting up a replication slave, see README.Debian about # other settings you may need to change. @@ -74,50 +74,35 @@ log_error = /var/log/mysql/error.log #log_bin = /var/log/mysql/mysql-bin.log expire_logs_days = 10 #max_binlog_size = 100M -#binlog_do_db = include_database_name -#binlog_ignore_db = exclude_database_name # -# * Security Features -# -# Read the manual, too, if you want chroot! -#chroot = /var/lib/mysql/ -# -# For generating SSL certificates you can use for example the GUI tool "tinyca". +# * SSL/TLS # + +# For documentation, please read +# https://mariadb.com/kb/en/securing-connections-for-client-and-server/ #ssl-ca = /etc/mysql/cacert.pem #ssl-cert = /etc/mysql/server-cert.pem #ssl-key = /etc/mysql/server-key.pem -# -# Accept only connections using the latest and most secure TLS protocol version. -# ..when MariaDB is compiled with OpenSSL: -#ssl-cipher = TLSv1.2 -# ..when MariaDB is compiled with YaSSL (default in Debian): -#ssl = on # # * Character sets # + # MySQL/MariaDB default is Latin1, but in Debian we rather default to the full # utf8 4-byte character set. See also client.cnf -# character-set-server = utf8mb4 collation-server = utf8mb4_general_ci # # * InnoDB # + # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. # Read the manual for more InnoDB related options. There are many! - -# -# * Unix socket authentication plugin is built-in since 10.0.22-6 -# -# Needed so the root database user can authenticate without a password but -# only when running as the unix root user. -# -# Also available for other users if required. -# See https://mariadb.com/kb/en/unix_socket-authentication-plugin/ +# Most important is to give InnoDB 80 % of the system RAM for buffer use: +# https://mariadb.com/kb/en/innodb-system-variables/#innodb_buffer_pool_size +#innodb_buffer_pool_size = 8G # this is only for embedded server [embedded] diff --git a/debian/additions/mariadb.conf.d/60-galera.cnf b/debian/additions/mariadb.conf.d/60-galera.cnf index 7249a0f1ec8..945c05281c9 100644 --- a/debian/additions/mariadb.conf.d/60-galera.cnf +++ b/debian/additions/mariadb.conf.d/60-galera.cnf @@ -1,6 +1,8 @@ # # * Galera-related settings # +# See the examples of server wsrep.cnf files in /usr/share/mysql + [galera] # Mandatory settings #wsrep_on=ON @@ -9,11 +11,10 @@ #binlog_format=row #default_storage_engine=InnoDB #innodb_autoinc_lock_mode=2 -# + # Allow server to accept connections on all interfaces. -# #bind-address=0.0.0.0 -# -# Optional setting + +# Optional settings #wsrep_slave_threads=1 #innodb_flush_log_at_trx_commit=0 diff --git a/debian/mariadb-server-10.5.install b/debian/mariadb-server-10.5.install index ca100506753..264949cf220 100644 --- a/debian/mariadb-server-10.5.install +++ b/debian/mariadb-server-10.5.install @@ -104,4 +104,5 @@ usr/share/man/man1/wsrep_sst_mysqldump.1 usr/share/man/man1/wsrep_sst_rsync.1 usr/share/man/man1/wsrep_sst_rsync_wan.1 usr/share/mysql/errmsg-utf8.txt +usr/share/mysql/wsrep.cnf usr/share/mysql/wsrep_notify From cc0205cf86220a2f2427336b34c19bbd66390038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Sun, 26 Apr 2020 22:43:57 +0300 Subject: [PATCH 6/6] MDEV-19917: Install Spider with a simple spider.cnf To install Spider one can simply drop a /etc/mysql/conf.d/spider.cnf like [mariadb] plugin-load-add=ha_spider.so This is automatically generated and installed when plugin is correctly registered to plugin.cmake with its own component name. Many other plugins such as Connect and RocksDB install in the same way. This solved MDEV-19917 as the mere adding and removing of spider.cnf automatically installs and uninstalls it. Remove the overly complex and uncecessary install.sql from Spider, if should not be needed in modern times anymore. With this change there is no need for a uninstall.sql either. --- debian/mariadb-plugin-spider.install | 2 +- debian/mariadb-plugin-spider.postinst | 10 ------ storage/spider/CMakeLists.txt | 7 ++-- storage/spider/scripts/install_spider.sql | 42 ----------------------- 4 files changed, 3 insertions(+), 58 deletions(-) delete mode 100644 debian/mariadb-plugin-spider.postinst delete mode 100644 storage/spider/scripts/install_spider.sql diff --git a/debian/mariadb-plugin-spider.install b/debian/mariadb-plugin-spider.install index 89652fe2f3c..78f2c44cbd0 100644 --- a/debian/mariadb-plugin-spider.install +++ b/debian/mariadb-plugin-spider.install @@ -1,2 +1,2 @@ +etc/mysql/conf.d/spider.cnf etc/mysql/mariadb.conf.d usr/lib/mysql/plugin/ha_spider.so -usr/share/mysql/install_spider.sql diff --git a/debian/mariadb-plugin-spider.postinst b/debian/mariadb-plugin-spider.postinst deleted file mode 100644 index d4ddaa53156..00000000000 --- a/debian/mariadb-plugin-spider.postinst +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -e - -# Install Spider -mysql --defaults-file=/etc/mysql/debian.cnf < /usr/share/mysql/install_spider.sql || true -# Always exit with success instead of leaving dpkg in a broken state - - -#DEBHELPER# diff --git a/storage/spider/CMakeLists.txt b/storage/spider/CMakeLists.txt index 706b11ac141..2c3573c3709 100644 --- a/storage/spider/CMakeLists.txt +++ b/storage/spider/CMakeLists.txt @@ -50,11 +50,8 @@ ELSEIF(PLUGIN_PARTITION MATCHES "^NO$") ELSE() INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/storage/spider/hs_client) - INSTALL(FILES - ${CMAKE_SOURCE_DIR}/storage/spider/scripts/install_spider.sql - DESTINATION ${INSTALL_MYSQLSHAREDIR} COMPONENT Server - ) - MYSQL_ADD_PLUGIN(spider ${SPIDER_SOURCES} STORAGE_ENGINE MODULE_ONLY MODULE_OUTPUT_NAME "ha_spider") + MYSQL_ADD_PLUGIN(spider ${SPIDER_SOURCES} + STORAGE_ENGINE COMPONENT spider-engine MODULE_ONLY MODULE_OUTPUT_NAME "ha_spider") IF(NOT TARGET spider) RETURN() ENDIF() diff --git a/storage/spider/scripts/install_spider.sql b/storage/spider/scripts/install_spider.sql deleted file mode 100644 index 403bd99fd68..00000000000 --- a/storage/spider/scripts/install_spider.sql +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (C) 2010-2019 Kentoku Shiba -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA - -drop procedure if exists mysql.spider_plugin_installer; -delimiter // -create procedure mysql.spider_plugin_installer() -begin - set @win_plugin := IF(@@version_compile_os like 'Win%', 1, 0); - set @have_spider_i_s_plugin := 0; - select @have_spider_i_s_plugin := 1 from INFORMATION_SCHEMA.plugins where PLUGIN_NAME = 'SPIDER'; - set @have_spider_plugin := 0; - select @have_spider_plugin := 1 from mysql.plugin where name = 'spider'; - if @have_spider_i_s_plugin = 0 then - if @have_spider_plugin = 1 then - -- spider plugin is present in mysql.plugin but not in - -- information_schema.plugins. Remove spider plugin entry - -- in mysql.plugin first. - delete from mysql.plugin where name = 'spider'; - end if; - -- Install spider plugin - if @win_plugin = 0 then - install plugin spider soname 'ha_spider.so'; - else - install plugin spider soname 'ha_spider.dll'; - end if; - end if; -end;// -delimiter ; -call mysql.spider_plugin_installer; -drop procedure mysql.spider_plugin_installer;