Merge branch '11.4' into 11.5

.clang-format

@@ -70,7 +70,6 @@ IndentPPDirectives: None
 IndentWidth:     2
 IndentWrappedFunctionNames: false
 KeepEmptyLinesAtTheStartOfBlocks: true
-Language: Cpp
 MacroBlockBegin: ''
 MacroBlockEnd:   ''
 MaxEmptyLinesToKeep: 1

.gitlab-ci.yml

@@ -508,9 +508,9 @@ mini-benchmark:
     - |
       mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
       grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly"
-    - yum install -y sysbench procps-ng perf util-linux || yum install -y https://kojipkgs.fedoraproject.org//packages/luajit/2.0.4/3.el7/x86_64/luajit-2.0.4-3.el7.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/sysbench/1.0.17/2.el7/x86_64/sysbench-1.0.17-2.el7.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/ck/0.5.2/2.el7/x86_64/ck-0.5.2-2.el7.x86_64.rpm
+    - yum install -y sysbench procps-ng perf flamegraph flamegraph-stackcollapse-perf util-linux dnf-utils
     - /usr/share/mariadb/mini-benchmark
-    - cp -av */sysbench-run-*.log */metrics.txt ..  # Move files one level down so they can be saved as artifacts
+    - cp -av */sysbench-run-*.log */metrics.txt .  # Move files one level down so they can be saved as artifacts
   artifacts:
     when: always
     paths:

BUILD/SETUP.sh

@@ -267,6 +267,12 @@ if test `$CC -v 2>&1 | tail -1 | sed 's/ .*$//'` = 'gcc' ; then
   fi
 fi
 
+if test `$CC -v 2>&1 | head -1 | sed 's/ .*$//'` = 'clang' ; then
+    dbug_cflags="$dbug_cflags -Wframe-larger-than=16384 -fno-inline"
+    c_warnings="$c_warnings -Wframe-larger-than=16384"
+    cxx_warnings="$cxx_warnings -Wframe-larger-than=16384"
+fi
+
 
 # If ccache (a compiler cache which reduces build time)
 # (http://samba.org/ccache) is installed, use it.

CMakeLists.txt

@@ -31,7 +31,7 @@ ENDIF()
 # in RPM's:
 
 #set(CPACK_RPM_SPEC_MORE_DEFINE "%define __spec_install_post /bin/true")
-FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0075 CMP0069 CMP0135)
+FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0074 CMP0075 CMP0069 CMP0135)
   IF(POLICY ${p})
     CMAKE_POLICY(SET ${p} NEW)
   ENDIF()
@@ -188,7 +188,7 @@ ENDIF()
 OPTION (WITH_UNIT_TESTS "Compile MySQL with unit tests" ON)
 
 IF (WITHOUT_SERVER)
-  SET (SKIP_COMPONENTS "Server|IniFiles|SuportFiles|Readme")
+  SET (SKIP_COMPONENTS "Server|IniFiles|SupportFiles|Readme")
 ELSE()
   SET (SKIP_COMPONENTS "N-O-N-E")
 ENDIF()
@@ -200,8 +200,9 @@ OPTION(NOT_FOR_DISTRIBUTION "Allow linking with GPLv2-incompatible system librar
 #  Can be switched on only for debug build.
 #
 OPTION(WITH_PROTECT_STATEMENT_MEMROOT "Enable protection of statement's memory root after first SP/PS execution. Turned into account only for debug build" OFF)
-IF (CMAKE_BUILD_TYPE MATCHES "Debug" AND WITH_PROTECT_STATEMENT_MEMROOT)
+IF (WITH_PROTECT_STATEMENT_MEMROOT)
-  ADD_DEFINITIONS(-DPROTECT_STATEMENT_MEMROOT)
+  SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DPROTECT_STATEMENT_MEMROOT")
+  SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DPROTECT_STATEMENT_MEMROOT")
 ENDIF()
 
 INCLUDE(check_compiler_flag)
@@ -265,8 +266,6 @@ IF(SECURITY_HARDENED AND NOT WITH_ASAN AND NOT WITH_UBSAN AND NOT WITH_TSAN AND
   MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO)
 ENDIF()
 
-INCLUDE(wsrep)
-
 OPTION(WITH_DBUG_TRACE "Enable DBUG_ENTER()/DBUG_RETURN()/DBUG_PRINT()" ON)
 IF(WITH_DBUG_TRACE)
   FOREACH(LANG C CXX)
@@ -278,6 +277,11 @@ ENDIF()
 SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC")
 SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC")
 
+IF(CMAKE_COMPILER_IS_GNUCC AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "10")
+  # Enable extra checks when using a recent enough version of GNU libstdc++
+  SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_GLIBCXX_DEBUG -D_GLIBCXX_ASSERTIONS")
+ENDIF()
+
 OPTION(ENABLE_GCOV "Enable gcov (debug, Linux builds only)" OFF)
 IF (ENABLE_GCOV)
   MY_CHECK_AND_SET_COMPILER_FLAG("-DHAVE_gcov -fprofile-arcs -ftest-coverage -lgcov" DEBUG)
@@ -338,6 +342,8 @@ ELSEIF(TRASH_FREED_MEMORY MATCHES "AUTO" AND NOT WIN32 AND NOT WITH_VALGRIND AND
   SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DTRASH_FREED_MEMORY")
 ENDIF()
 
+INCLUDE(wsrep)
+
 # Set commonly used variables
 IF(WIN32)
   SET(DEFAULT_MYSQL_HOME "C:/Program Files/MariaDB ${MYSQL_BASE_VERSION}")

README.md

@@ -3,7 +3,7 @@ Code status:
 
 * [![Appveyor CI status](https://ci.appveyor.com/api/projects/status/4u6pexmtpuf8jq66?svg=true)](https://ci.appveyor.com/project/rasmushoj/server) ci.appveyor.com
 
-## MariaDB: The open source relational database 
+## MariaDB: The innovative open source database 
 
 MariaDB was designed as a drop-in replacement of MySQL(R) with more
 features, new storage engines, fewer bugs, and better performance.
@@ -33,28 +33,23 @@ https://mariadb.com/kb/en/mariadb-versus-mysql-compatibility/
 
 https://mariadb.com/kb/en/new-and-old-releases/
 
+Getting the code, building it and testing it
+---------------------------------------------------------------
+
+Refer to the following guide: https://mariadb.org/get-involved/getting-started-for-developers/get-code-build-test/ which outlines how to correctly build the source code and run the MariaDB testing framework.
+
 Help
 -----
 
 More help is available from the Maria Discuss mailing list
-https://launchpad.net/~maria-discuss, MariaDB's Zulip
+https://lists.mariadb.org/postorius/lists/discuss.lists.mariadb.org/ and MariaDB's Zulip
 instance, https://mariadb.zulipchat.com/ 
 
-Live QA for beginner contributors
-----
-MariaDB has a dedicated time each week when we answer new contributor questions live on Zulip.
-From 8:00 to 10:00 UTC on Mondays, and 10:00 to 12:00 UTC on Thursdays,
-anyone can ask any questions they’d like, and a live developer will be available to assist.
-
-New contributors can ask questions any time, but we will provide immediate feedback during that interval.
-
 Licensing
 ---------
 
 ***************************************************************************
 
-NOTE: 
-
 MariaDB is specifically available only under version 2 of the GNU
 General Public License (GPLv2). (I.e. Without the "any later version"
 clause.) This is inherited from MySQL. Please see the README file in

client/CMakeLists.txt

@@ -18,7 +18,7 @@ INCLUDE_DIRECTORIES(
   ${CMAKE_SOURCE_DIR}/include
   ${PCRE_INCLUDE_DIRS}
   ${CMAKE_SOURCE_DIR}/mysys_ssl
-  ${ZLIB_INCLUDE_DIR}
+  ${ZLIB_INCLUDE_DIRS}
   ${SSL_INCLUDE_DIRS}
   ${CMAKE_SOURCE_DIR}/sql
   ${CMAKE_SOURCE_DIR}/strings

client/client_priv.h

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2001, 2012, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2022, MariaDB
+   Copyright (c) 2009, 2024, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -38,75 +38,35 @@ enum options_client
 {
   OPT_CHARSETS_DIR=256, OPT_DEFAULT_CHARSET,
   OPT_PAGER, OPT_TEE,
-  OPT_LOW_PRIORITY, OPT_AUTO_REPAIR, OPT_COMPRESS,
+  OPT_OPTIMIZE,
-  OPT_DROP, OPT_LOCKS, OPT_KEYWORDS, OPT_DELAYED, OPT_OPTIMIZE,
+  OPT_TABLES,
-  OPT_FTB, OPT_LTB, OPT_ENC, OPT_O_ENC, OPT_ESC, OPT_TABLES,
+  OPT_MASTER_DATA,
-  OPT_MASTER_DATA, OPT_AUTOCOMMIT, OPT_AUTO_REHASH,
-  OPT_LINE_NUMBERS, OPT_COLUMN_NAMES, OPT_CONNECT_TIMEOUT,
-  OPT_MAX_ALLOWED_PACKET, OPT_NET_BUFFER_LENGTH,
-  OPT_SELECT_LIMIT, OPT_MAX_JOIN_SIZE, OPT_SSL_SSL,
   OPT_SSL_KEY, OPT_SSL_CERT, OPT_SSL_CA, OPT_SSL_CAPATH,
-  OPT_SSL_CIPHER, OPT_TLS_VERSION, OPT_SHUTDOWN_TIMEOUT, OPT_LOCAL_INFILE,
+  OPT_TLS_VERSION,
-  OPT_DELETE_MASTER_LOGS, OPT_COMPACT,
+  OPT_SSL_CIPHER, OPT_LOCAL_INFILE,
-  OPT_PROMPT, OPT_IGN_LINES,OPT_TRANSACTION,OPT_MYSQL_PROTOCOL,
+  OPT_COMPACT,
-  OPT_FRM, OPT_SKIP_OPTIMIZATION,
+  OPT_MYSQL_PROTOCOL,
-  OPT_COMPATIBLE, OPT_RECONNECT, OPT_DELIMITER, OPT_SECURE_AUTH,
+  OPT_SKIP_OPTIMIZATION,
-  OPT_OPEN_FILES_LIMIT, OPT_SET_CHARSET, OPT_SERVER_ARG,
+  OPT_COMPATIBLE, OPT_DELIMITER,
-  OPT_STOP_POSITION, OPT_START_DATETIME, OPT_STOP_DATETIME,
+  OPT_SERVER_ARG,
-  OPT_SIGINT_IGNORE, OPT_HEXBLOB, OPT_ORDER_BY_PRIMARY, OPT_COUNT,
+  OPT_START_DATETIME, OPT_STOP_DATETIME,
-  OPT_FLUSH_TABLES,
-  OPT_TRIGGERS,
-  OPT_MYSQL_ONLY_PRINT,
-  OPT_MYSQL_LOCK_DIRECTORY,
-  OPT_USE_THREADS,
-  OPT_IMPORT_USE_THREADS,
-  OPT_MYSQL_NUMBER_OF_QUERY,
   OPT_IGNORE_DATABASE,
-  OPT_IGNORE_TABLE,OPT_INSERT_IGNORE,OPT_SHOW_WARNINGS,OPT_DROP_DATABASE,
+  OPT_IGNORE_TABLE,
-  OPT_TZ_UTC, OPT_CREATE_SLAP_SCHEMA,
-  OPT_MYSQLDUMP_SLAVE_APPLY,
   OPT_MYSQLDUMP_SLAVE_DATA,
-  OPT_MYSQLDUMP_INCLUDE_MASTER_HOST_PORT,
+  OPT_SLAP_CSV,
-#ifdef WHEN_FLASHBACK_REVIEW_READY
+  OPT_BASE64_OUTPUT_MODE,
-  OPT_REVIEW,
+  OPT_FIX_TABLE_NAMES, OPT_FIX_DB_NAMES,
-  OPT_REVIEW_DBNAME, OPT_REVIEW_TABLENAME,
+  OPT_WRITE_BINLOG,
-#endif
-  OPT_SLAP_CSV, OPT_SLAP_CREATE_STRING,
-  OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, OPT_SLAP_AUTO_GENERATE_WRITE_NUM,
-  OPT_SLAP_AUTO_GENERATE_ADD_AUTO,
-  OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY,
-  OPT_SLAP_AUTO_GENERATE_EXECUTE_QUERIES,
-  OPT_SLAP_AUTO_GENERATE_SECONDARY_INDEXES,
-  OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM,
-  OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM,
-  OPT_SLAP_PRE_QUERY,
-  OPT_SLAP_POST_QUERY,
-  OPT_SLAP_PRE_SYSTEM,
-  OPT_SLAP_POST_SYSTEM,
-  OPT_SLAP_COMMIT,
-  OPT_SLAP_DETACH,
-  OPT_SLAP_NO_DROP,
-  OPT_MYSQL_REPLACE_INTO, OPT_BASE64_OUTPUT_MODE, OPT_SERVER_ID,
-  OPT_FIX_TABLE_NAMES, OPT_FIX_DB_NAMES, OPT_SSL_VERIFY_SERVER_CERT,
-  OPT_AUTO_VERTICAL_OUTPUT,
-  OPT_DEBUG_INFO, OPT_DEBUG_CHECK, OPT_COLUMN_TYPES, OPT_ERROR_LOG_FILE,
-  OPT_WRITE_BINLOG, OPT_DUMP_DATE,
-  OPT_INIT_COMMAND,
   OPT_PLUGIN_DIR,
   OPT_DEFAULT_AUTH,
-  OPT_ABORT_SOURCE_ON_ERROR,
   OPT_REWRITE_DB,
-  OPT_REPORT_PROGRESS,
-  OPT_SKIP_ANNOTATE_ROWS_EVENTS,
   OPT_SSL_CRL, OPT_SSL_CRLPATH,
   OPT_IGNORE_DATA,
   OPT_PRINT_ROW_COUNT, OPT_PRINT_ROW_EVENT_POSITIONS,
   OPT_CHECK_IF_UPGRADE_NEEDED,
   OPT_COMPATIBILTY_CLEARTEXT_PLUGIN,
-  OPT_SHUTDOWN_WAIT_FOR_SLAVES,
+  OPT_STOP_POSITION,
-  OPT_COPY_S3_TABLES,
+  OPT_SERVER_ID,
-  OPT_PRINT_TABLE_METADATA,
-  OPT_ASOF_TIMESTAMP,
   OPT_IGNORE_DOMAIN_IDS,
   OPT_DO_DOMAIN_IDS,
   OPT_IGNORE_SERVER_IDS,

client/mysql.cc

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2018, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2022, MariaDB Corporation.
+   Copyright (c) 2009, 2024, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -226,7 +226,7 @@ typedef struct st_status
   ulong query_start_line;
   char *file_name;
   LINE_BUFFER *line_buff;
-  bool batch,add_to_history;
+  bool batch, add_to_history, sandbox;
 } STATUS;
 
 
@@ -324,7 +324,8 @@ static int com_quit(String *str,char*),
 	   com_rehash(String *str, char*), com_tee(String *str, char*),
            com_notee(String *str, char*), com_charset(String *str,char*),
            com_prompt(String *str, char*), com_delimiter(String *str, char*),
-     com_warnings(String *str, char*), com_nowarnings(String *str, char*);
+     com_warnings(String *str, char*), com_nowarnings(String *str, char*),
+     com_sandbox(String *str, char*);
 static int com_query_cost(String *str, char*);
 
 #ifdef USE_POPEN
@@ -373,11 +374,12 @@ typedef struct {
 
 static COMMANDS commands[] = {
   { "?",      '?', com_help,   1, "Synonym for `help'." },
+  { "charset",    'C', com_charset,    1,
+    "Switch to another charset. Might be needed for processing binlog with multi-byte charsets." },
   { "clear",  'c', com_clear,  0, "Clear the current input statement."},
   { "connect",'r', com_connect,1,
     "Reconnect to the server. Optional arguments are db and host." },
-  { "delimiter", 'd', com_delimiter,    1,
+  { "delimiter", 'd', com_delimiter,    1, "Set statement delimiter." },
-    "Set statement delimiter." },
 #ifdef USE_POPEN
   { "edit",   'e', com_edit,   0, "Edit command with $EDITOR."},
 #endif
@@ -390,6 +392,8 @@ static COMMANDS commands[] = {
   { "nopager",'n', com_nopager,0, "Disable pager, print to stdout." },
 #endif
   { "notee",  't', com_notee,  0, "Don't write into outfile." },
+  { "nowarning", 'w', com_nowarnings, 0,
+    "Don't show warnings after every statement." },
 #ifdef USE_POPEN
   { "pager",  'P', com_pager,  1, 
     "Set PAGER [to_pager]. Print the query results via PAGER." },
@@ -400,6 +404,8 @@ static COMMANDS commands[] = {
   { "costs",  'Q', com_query_cost,  0,
     "Toggle showing query costs after each query" },
   { "rehash", '#', com_rehash, 0, "Rebuild completion hash." },
+  { "sandbox", '-', com_sandbox, 0,
+    "Disallow commands that access the file system (except \\P without an argument and \\e)." },
   { "source", '.', com_source, 1,
     "Execute an SQL script file. Takes a file name as an argument."},
   { "status", 's', com_status, 0, "Get status information from the server."},
@@ -410,12 +416,8 @@ static COMMANDS commands[] = {
     "Set outfile [to_outfile]. Append everything into given outfile." },
   { "use",    'u', com_use,    1,
     "Use another database. Takes database name as argument." },
-  { "charset",    'C', com_charset,    1,
-    "Switch to another charset. Might be needed for processing binlog with multi-byte charsets." },
   { "warnings", 'W', com_warnings,  0,
     "Show warnings after every statement." },
-  { "nowarning", 'w', com_nowarnings, 0,
-    "Don't show warnings after every statement." },
   /* Get bash-like expansion for some commands */
   { "create table",     0, 0, 0, ""},
   { "create database",  0, 0, 0, ""},
@@ -1642,35 +1644,47 @@ static struct my_option my_long_options[] =
    0, 0, 0, 0, 0},
   {"help", 'I', "Synonym for -?", 0, 0, 0, GET_NO_ARG, NO_ARG, 0,
    0, 0, 0, 0, 0},
-  {"abort-source-on-error", OPT_ABORT_SOURCE_ON_ERROR,
+  {"abort-source-on-error", 0,
    "Abort 'source filename' operations in case of errors",
    &batch_abort_on_error, &batch_abort_on_error, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"auto-rehash", OPT_AUTO_REHASH,
+  {"auto-rehash", 0,
    "Enable automatic rehashing. One doesn't need to use 'rehash' to get table "
-   "and field completion, but startup and reconnecting may take a longer time. "
+   "and field completion, but startup and reconnecting may take a longer time.",
-   "Disable with --disable-auto-rehash.",
+   &opt_rehash, &opt_rehash, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-   &opt_rehash, &opt_rehash, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0,
-   0, 0},
   {"no-auto-rehash", 'A',
    "No automatic rehashing. One has to use 'rehash' to get table and field "
    "completion. This gives a quicker start of mysql and disables rehashing "
    "on reconnect.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
-   {"auto-vertical-output", OPT_AUTO_VERTICAL_OUTPUT,
+   {"auto-vertical-output", 0,
     "Automatically switch to vertical output mode if the result is wider "
-    "than the terminal width.",
+    "than the terminal width.", &auto_vertical_output, &auto_vertical_output,
-    &auto_vertical_output, &auto_vertical_output, 0, GET_BOOL, NO_ARG, 0,
+    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-    0, 0, 0, 0, 0},
   {"batch", 'B',
    "Don't use history file. Disable interactive behavior. (Enables --silent.)",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"binary-as-hex", 0, "Print binary data as hex", &opt_binhex, &opt_binhex,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"binary-mode", 0,
+   "Binary mode allows certain character sequences to be processed as data "
+   "that would otherwise be treated with a special meaning by the parser. "
+   "Specifically, this switch turns off parsing of all client commands except "
+   "\\C and DELIMITER in non-interactive mode (i.e., when binary mode is "
+   "combined with either 1) piped input, 2) the --batch mysql option, or 3) "
+   "the 'source' command). Also, in binary mode, occurrences of '\\r\\n' and "
+   "ASCII '\\0' are preserved within strings, whereas by default, '\\r\\n' is "
+   "translated to '\\n' and '\\0' is disallowed in user input.",
+   &opt_binary_mode, &opt_binary_mode, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"character-sets-dir", OPT_CHARSETS_DIR,
    "Directory for character set files.", &charsets_dir,
    &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"column-type-info", OPT_COLUMN_TYPES, "Display column type information.",
+  {"column-names", 0, "Write column names in results.",
+   &column_names, &column_names, 0, GET_BOOL,
+   NO_ARG, 1, 0, 0, 0, 0, 0},
+  {"skip-column-names", 'N', "Don't write column names in results.", 0, 0, 0,
+    GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"column-type-info", 0, "Display column type information.",
    &column_types_flag, &column_types_flag,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"comments", 'c', "Preserve comments. Send comments to the server."
@@ -1680,6 +1694,16 @@ static struct my_option my_long_options[] =
   {"compress", 'C', "Use compression in server/client protocol.",
    &opt_compress, &opt_compress, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
+  {"connect-expired-password", 0,
+   "Notify the server that this client is prepared to handle expired "
+   "password sandbox mode even if --batch was specified.",
+   &opt_connect_expired_password, &opt_connect_expired_password, 0, GET_BOOL,
+   NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"connect_timeout", 0, "Number of seconds before connection timeout.",
+   &opt_connect_timeout, &opt_connect_timeout, 0, GET_ULONG, REQUIRED_ARG,
+   0, 0, 3600*12, 0, 0, 0},
+  {"database", 'D', "Database to use.", &current_db,
+   &current_db, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 #ifdef DBUG_OFF
   {"debug", '#', "This is a non-debug version. Catch this and exit.",
    0,0, 0, GET_DISABLED, OPT_ARG, 0, 0, 0, 0, 0, 0},
@@ -1687,70 +1711,64 @@ static struct my_option my_long_options[] =
   {"debug", '#', "Output debug log.", &default_dbug_option,
    &default_dbug_option, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit.",
+  {"debug-check", 0, "Check memory and open file usage at exit.",
    &debug_check_flag, &debug_check_flag, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"debug-info", 'T', "Print some debug info at exit.", &debug_info_flag,
    &debug_info_flag, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"database", 'D', "Database to use.", &current_db,
+  {"default-auth", 0, "Default authentication client-side plugin to use.",
-   &current_db, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+    &opt_default_auth, &opt_default_auth, 0,
-  {"default-character-set", OPT_DEFAULT_CHARSET,
+   GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"default-character-set", 0,
    "Set the default character set.", &default_charset,
    &default_charset, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"delimiter", OPT_DELIMITER, "Delimiter to be used.", &delimiter_str,
    &delimiter_str, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"enable-cleartext-plugin", OPT_COMPATIBILTY_CLEARTEXT_PLUGIN,
+   "Obsolete option. Exists only for MySQL compatibility.",
+   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"execute", 'e', "Execute command and quit. (Disables --force and history file.)", 0,
    0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"enable-cleartext-plugin", OPT_COMPATIBILTY_CLEARTEXT_PLUGIN, "Obsolete option. Exists only for MySQL compatibility.",
+  {"force", 'f',
-   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+    "Continue even if we get an SQL error. Sets abort-source-on-error to 0",
-  {"vertical", 'E', "Print the output of a query (rows) vertically.",
+    &ignore_errors, &ignore_errors, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   &vertical, &vertical, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0,
+  {"host", 'h', "Connect to host.", &current_host,
-   0},
+   &current_host, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"force", 'f', "Continue even if we get an SQL error. Sets abort-source-on-error to 0",
+  {"html", 'H', "Produce HTML output.", &opt_html, &opt_html,
-   &ignore_errors, &ignore_errors, 0, GET_BOOL, NO_ARG, 0, 0,
+   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0},
+  {"ignore-spaces", 'i', "Ignore space after function names.",
+   &ignore_spaces, &ignore_spaces, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"init-command", 0,
+   "SQL Command to execute when connecting to MariaDB server. Will "
+   "automatically be re-executed when reconnecting.", &opt_init_command,
+   &opt_init_command, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"line-numbers", 0, "Write line numbers for errors.",
+   &line_numbers, &line_numbers, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
+  {"skip-line-numbers", 'L', "Don't write line number for errors.", 0, 0, 0,
+    GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"local-infile", OPT_LOCAL_INFILE, "Enable LOAD DATA LOCAL INFILE.",
+   &opt_local_infile, &opt_local_infile, 0, GET_BOOL, OPT_ARG, 0, 0, 0, 0, 0, 0},
+  {"max-allowed-packet", 0,
+   "The maximum packet length to send to or receive from server.",
+   &opt_max_allowed_packet, &opt_max_allowed_packet, 0, GET_ULONG,
+   REQUIRED_ARG, 16*1024LL*1024LL, 4096, 2*1024LL*1024LL*1024LL, 0, 1024, 0},
+  {"max-join-size", 0,
+   "Automatic limit for rows in a join when using --safe-updates.",
+   &max_join_size, &max_join_size, 0, GET_ULONG, REQUIRED_ARG, 1000000L,
+   1, ULONG_MAX, 0, 1, 0},
   {"named-commands", 'G',
    "Enable named commands. Named commands mean this program's internal "
    "commands; see mysql> help . When enabled, the named commands can be "
    "used from any line of the query, otherwise only from the first line, "
    "before an enter. Disable with --disable-named-commands. This option "
    "is disabled by default.",
-   &named_cmds, &named_cmds, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+   &named_cmds, &named_cmds, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0},
+  {"net-buffer-length", 0,
-  {"ignore-spaces", 'i', "Ignore space after function names.",
+   "The buffer size for TCP/IP and socket communication.",
-   &ignore_spaces, &ignore_spaces, 0, GET_BOOL, NO_ARG, 0, 0,
+   &opt_net_buffer_length, &opt_net_buffer_length, 0, GET_ULONG,
-   0, 0, 0, 0},
+   REQUIRED_ARG, 16384, 1024, 512*1024ULL*1024ULL, MALLOC_OVERHEAD, 1024, 0},
-  {"init-command", OPT_INIT_COMMAND,
-   "SQL Command to execute when connecting to MariaDB server. Will "
-   "automatically be re-executed when reconnecting.",
-   &opt_init_command, &opt_init_command, 0,
-   GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"local-infile", OPT_LOCAL_INFILE, "Enable/disable LOAD DATA LOCAL INFILE.",
-   &opt_local_infile, &opt_local_infile, 0, GET_BOOL, OPT_ARG, 0, 0, 0, 0, 0, 0},
   {"no-beep", 'b', "Turn off beep on error.", &opt_nobeep,
    &opt_nobeep, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"host", 'h', "Connect to host.", &current_host,
-   &current_host, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"html", 'H', "Produce HTML output.", &opt_html, &opt_html,
-   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"xml", 'X', "Produce XML output.", &opt_xml, &opt_xml, 0,
-   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"line-numbers", OPT_LINE_NUMBERS, "Write line numbers for errors.",
-   &line_numbers, &line_numbers, 0, GET_BOOL,
-   NO_ARG, 1, 0, 0, 0, 0, 0},
-  {"skip-line-numbers", 'L', "Don't write line number for errors.", 0, 0, 0, GET_NO_ARG,
-   NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"unbuffered", 'n', "Flush buffer after each query.", &unbuffered,
-   &unbuffered, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"column-names", OPT_COLUMN_NAMES, "Write column names in results.",
-   &column_names, &column_names, 0, GET_BOOL,
-   NO_ARG, 1, 0, 0, 0, 0, 0},
-  {"skip-column-names", 'N',
-   "Don't write column names in results.",
-   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"sigint-ignore", OPT_SIGINT_IGNORE, "Ignore SIGINT (CTRL-C).",
-   &opt_sigint_ignore,  &opt_sigint_ignore, 0, GET_BOOL,
-   NO_ARG, 0, 0, 0, 0, 0, 0},
   {"one-database", 'o',
    "Ignore statements except those that occur while the default "
    "database is the one named at the command line.",
@@ -1771,19 +1789,20 @@ static struct my_option my_long_options[] =
   {"pipe", 'W', "Use named pipes to connect to server.", 0, 0, 0, GET_NO_ARG,
    NO_ARG, 0, 0, 0, 0, 0, 0},
 #endif
+  {"plugin-dir", 0, "Directory for client-side plugins.", &opt_plugin_dir,
+    &opt_plugin_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"port", 'P', "Port number to use for connection or 0 for default to, in "
    "order of preference, my.cnf, $MYSQL_TCP_PORT, "
 #if MYSQL_PORT_DEFAULT == 0
    "/etc/services, "
 #endif
-   "built-in default (" STRINGIFY_ARG(MYSQL_PORT) ").",
+   "built-in default (" STRINGIFY_ARG(MYSQL_PORT) ").", &opt_mysql_port,
-   &opt_mysql_port,
    &opt_mysql_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"progress-reports", OPT_REPORT_PROGRESS,
+  {"progress-reports", 0,
    "Get progress reports for long running commands (like ALTER TABLE)",
    &opt_progress_reports, &opt_progress_reports, 0, GET_BOOL, NO_ARG, 1, 0,
    0, 0, 0, 0},
-  {"prompt", OPT_PROMPT, "Set the command line prompt to this value.",
+  {"prompt", 0, "Set the command line prompt to this value.",
    &current_prompt, &current_prompt, 0, GET_STR_ALLOC,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"protocol", OPT_MYSQL_PROTOCOL, "The protocol to use for connection (tcp, socket, pipe).",
@@ -1794,11 +1813,29 @@ static struct my_option my_long_options[] =
    "if the output is suspended. Doesn't use history file.",
    &quick, &quick, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"raw", 'r', "Write fields without conversion. Used with --batch.",
-   &opt_raw_data, &opt_raw_data, 0, GET_BOOL, NO_ARG, 0, 0, 0,
+   &opt_raw_data, &opt_raw_data, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0},
+  {"reconnect", 0, "Reconnect if the connection is lost.",
-  {"reconnect", OPT_RECONNECT, "Reconnect if the connection is lost. Disable "
-   "with --disable-reconnect. This option is enabled by default.",
    &opt_reconnect, &opt_reconnect, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
+  {"safe-updates", 'U', "Only allow UPDATE and DELETE that uses keys.",
+   &safe_updates, &safe_updates, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"i-am-a-dummy", 'U', "Synonym for option --safe-updates, -U.",
+   &safe_updates, &safe_updates, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"sandbox", 0, "Disallow commands that access the file system (except \\P without an argument and \\e).",
+   &status.sandbox, &status.sandbox, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"secure-auth", 0, "Refuse client connecting to server if it"
+    " uses old (pre-4.1.1) protocol.", &opt_secure_auth,
+    &opt_secure_auth, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"select-limit", 0,
+   "Automatic limit for SELECT when using --safe-updates.", &select_limit,
+   &select_limit, 0, GET_ULONG, REQUIRED_ARG, 1000L, 1, ULONG_MAX, 0, 1, 0},
+  {"server-arg", OPT_SERVER_ARG, "Send embedded server this as a parameter.",
+   0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"show-query-costs", 0, "Show query cost after every statement.",
+    &show_query_cost, &show_query_cost, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"show-warnings", 0, "Show warnings after every statement.",
+    &show_warnings, &show_warnings, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"sigint-ignore", 0, "Ignore SIGINT (CTRL-C).", &opt_sigint_ignore,
+    &opt_sigint_ignore, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"silent", 's', "Be more silent. Print results with a tab as separator, "
    "each row on new line.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"socket", 'S', "The socket file to use for connection.",
@@ -1812,77 +1849,22 @@ static struct my_option my_long_options[] =
    "Does not work in batch mode. Disable with --disable-tee. "
    "This option is disabled by default.",
    0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"unbuffered", 'n', "Flush buffer after each query.", &unbuffered,
+   &unbuffered, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 #ifndef DONT_ALLOW_USER_CHANGE
   {"user", 'u', "User for login if not current user.", &current_user,
    &current_user, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"safe-updates", 'U', "Only allow UPDATE and DELETE that uses keys.",
-   &safe_updates, &safe_updates, 0, GET_BOOL, NO_ARG, 0, 0,
-   0, 0, 0, 0},
-  {"i-am-a-dummy", 'U', "Synonym for option --safe-updates, -U.",
-   &safe_updates, &safe_updates, 0, GET_BOOL, NO_ARG, 0, 0,
-   0, 0, 0, 0},
   {"verbose", 'v', "Write more. (-v -v -v gives the table output format).", 0,
    0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"version", 'V', "Output version information and exit.", 0, 0, 0,
    GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"vertical", 'E', "Print the output of a query (rows) vertically.",
+   &vertical, &vertical, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"wait", 'w', "Wait and retry if connection is down.", 0, 0, 0, GET_NO_ARG,
    NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"connect_timeout", OPT_CONNECT_TIMEOUT,
+  {"xml", 'X', "Produce XML output.", &opt_xml, &opt_xml, 0,
-   "Number of seconds before connection timeout.",
+   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   &opt_connect_timeout, &opt_connect_timeout, 0, GET_ULONG, REQUIRED_ARG,
-   0, 0, 3600*12, 0, 0, 0},
-  {"max_allowed_packet", OPT_MAX_ALLOWED_PACKET,
-   "The maximum packet length to send to or receive from server.",
-   &opt_max_allowed_packet, &opt_max_allowed_packet, 0,
-   GET_ULONG, REQUIRED_ARG, 16 *1024L*1024L, 4096,
-   (longlong) 2*1024L*1024L*1024L, MALLOC_OVERHEAD, 1024, 0},
-  {"net_buffer_length", OPT_NET_BUFFER_LENGTH,
-   "The buffer size for TCP/IP and socket communication.",
-   &opt_net_buffer_length, &opt_net_buffer_length, 0, GET_ULONG,
-   REQUIRED_ARG, 16384, 1024, 512*1024*1024L, MALLOC_OVERHEAD, 1024, 0},
-  {"select_limit", OPT_SELECT_LIMIT,
-   "Automatic limit for SELECT when using --safe-updates.",
-   &select_limit, &select_limit, 0, GET_ULONG, REQUIRED_ARG, 1000L,
-   1, ULONG_MAX, 0, 1, 0},
-  {"max_join_size", OPT_MAX_JOIN_SIZE,
-   "Automatic limit for rows in a join when using --safe-updates.",
-   &max_join_size, &max_join_size, 0, GET_ULONG, REQUIRED_ARG, 1000000L,
-   1, ULONG_MAX, 0, 1, 0},
-  {"secure-auth", OPT_SECURE_AUTH, "Refuse client connecting to server if it"
-    " uses old (pre-4.1.1) protocol.", &opt_secure_auth,
-    &opt_secure_auth, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"server-arg", OPT_SERVER_ARG, "Send embedded server this as a parameter.",
-   0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"show-warnings", OPT_SHOW_WARNINGS, "Show warnings after every statement.",
-    &show_warnings, &show_warnings, 0, GET_BOOL, NO_ARG,
-    0, 0, 0, 0, 0, 0},
-  {"show-query-costs", OPT_SHOW_WARNINGS,
-   "Show query cost after every statement.",
-    &show_query_cost, &show_query_cost, 0, GET_BOOL, NO_ARG,
-    0, 0, 0, 0, 0, 0},
-  {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
-    &opt_plugin_dir, &opt_plugin_dir, 0,
-   GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"default_auth", OPT_DEFAULT_AUTH,
-    "Default authentication client-side plugin to use.",
-    &opt_default_auth, &opt_default_auth, 0,
-   GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"binary-mode", 0,
-   "Binary mode allows certain character sequences to be processed as data "
-   "that would otherwise be treated with a special meaning by the parser. "
-   "Specifically, this switch turns off parsing of all client commands except "
-   "\\C and DELIMITER in non-interactive mode (i.e., when binary mode is "
-   "combined with either 1) piped input, 2) the --batch mysql option, or 3) "
-   "the 'source' command). Also, in binary mode, occurrences of '\\r\\n' and "
-   "ASCII '\\0' are preserved within strings, whereas by default, '\\r\\n' is "
-   "translated to '\\n' and '\\0' is disallowed in user input.",
-   &opt_binary_mode, &opt_binary_mode, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"connect-expired-password", 0,
-   "Notify the server that this client is prepared to handle expired "
-   "password sandbox mode even if --batch was specified.",
-   &opt_connect_expired_password, &opt_connect_expired_password, 0, GET_BOOL,
-   NO_ARG, 0, 0, 0, 0, 0, 0},
   { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -2934,9 +2916,7 @@ static void initialize_readline ()
   array of matches, or NULL if there aren't any.
 */
 
-static char **new_mysql_completion(const char *text,
+static char **new_mysql_completion(const char *text, int, int)
-                                   int start __attribute__((unused)),
-                                   int end __attribute__((unused)))
 {
   if (!status.batch && !quick)
 #if defined(USE_NEW_READLINE_INTERFACE)
@@ -3259,8 +3239,7 @@ static void print_help_item(MYSQL_ROW *cur, int num_name, int num_cat, char *las
 }
 
 
-static int com_server_help(String *buffer __attribute__((unused)),
+static int com_server_help(String *buffer, char *, char *help_arg)
-			   char *line __attribute__((unused)), char *help_arg)
 {
   MYSQL_ROW cur;
   const char *server_cmd;
@@ -3362,9 +3341,7 @@ err:
   return error;
 }
 
-static int
+static int com_help(String *buffer, char *line)
-com_help(String *buffer __attribute__((unused)),
-	 char *line __attribute__((unused)))
 {
   int i, j;
   char * help_arg= strchr(line,' '), buff[32], *end;
@@ -3396,9 +3373,7 @@ com_help(String *buffer __attribute__((unused)),
 }
 
 
-	/* ARGSUSED */
+static int com_clear(String *buffer,char *)
-static int
-com_clear(String *buffer,char *line __attribute__((unused)))
 {
 #ifdef HAVE_READLINE
   if (status.add_to_history)
@@ -3423,9 +3398,7 @@ static void adjust_console_codepage(const char *name __attribute__((unused)))
 }
 
 
-	/* ARGSUSED */
+static int com_charset(String *, char *line)
-static int
-com_charset(String *buffer __attribute__((unused)), char *line)
 {
   char buff[256], *param;
   CHARSET_INFO * new_cs;
@@ -3458,8 +3431,7 @@ com_charset(String *buffer __attribute__((unused)), char *line)
 */
 
 
-static int
+static int com_go(String *buffer, char *)
-com_go(String *buffer,char *line __attribute__((unused)))
 {
   char		buff[200]; /* about 110 chars used so far */
   char		time_buff[53+3+1]; /* time max + space & parens + NUL */
@@ -4044,9 +4016,7 @@ tee_print_sized_data(const char *data, unsigned int data_length, unsigned int to
 }
 
 
-
+static void print_table_data_html(MYSQL_RES *result)
-static void
-print_table_data_html(MYSQL_RES *result)
 {
   MYSQL_ROW	cur;
   MYSQL_FIELD	*field;
@@ -4369,12 +4339,12 @@ print_tab_data(MYSQL_RES *result)
   }
 }
 
-static int
+static int com_tee(String *, char *line)
-com_tee(String *buffer __attribute__((unused)),
-        char *line __attribute__((unused)))
 {
   char file_name[FN_REFLEN], *end, *param;
 
+  if (status.sandbox)
+    return put_info("Not allowed in the sandbox mode", INFO_ERROR, 0);
   if (status.batch)
     return 0;
   while (my_isspace(charset_info, *line))
@@ -4414,9 +4384,7 @@ com_tee(String *buffer __attribute__((unused)),
 }
 
 
-static int
+static int com_notee(String *, char *)
-com_notee(String *buffer __attribute__((unused)),
-	  char *line __attribute__((unused)))
 {
   if (opt_outfile)
     end_tee();
@@ -4429,9 +4397,7 @@ com_notee(String *buffer __attribute__((unused)),
 */
 
 #ifdef USE_POPEN
-static int
+static int com_pager(String *, char *line)
-com_pager(String *buffer __attribute__((unused)),
-          char *line __attribute__((unused)))
 {
   char pager_name[FN_REFLEN], *end, *param;
 
@@ -4459,6 +4425,8 @@ com_pager(String *buffer __attribute__((unused)),
   }
   else
   {
+    if (status.sandbox)
+      return put_info("Not allowed in the sandbox mode", INFO_ERROR, 0);
     end= strmake_buf(pager_name, param);
     while (end > pager_name && (my_isspace(charset_info,end[-1]) || 
                                 my_iscntrl(charset_info,end[-1])))
@@ -4473,9 +4441,7 @@ com_pager(String *buffer __attribute__((unused)),
 }
 
 
-static int
+static int com_nopager(String *, char *)
-com_nopager(String *buffer __attribute__((unused)),
-	    char *line __attribute__((unused)))
 {
   strmov(pager, "stdout");
   opt_nopager=1;
@@ -4487,7 +4453,7 @@ com_nopager(String *buffer __attribute__((unused)),
 
 #ifdef USE_POPEN
 static int
-com_edit(String *buffer,char *line __attribute__((unused)))
+com_edit(String *buffer,char *)
 {
   char	filename[FN_REFLEN],buff[160];
   int	fd,tmp,error;
@@ -4534,17 +4500,15 @@ err:
 
 /* If arg is given, exit without errors. This happens on command 'quit' */
 
-static int
+static int com_quit(String *, char *)
-com_quit(String *buffer __attribute__((unused)),
-	 char *line __attribute__((unused)))
 {
   status.exit_status=0;
   return 1;
 }
 
 static int
-com_rehash(String *buffer __attribute__((unused)),
+com_rehash(String *,
-	 char *line __attribute__((unused)))
+	 char *)
 {
 #ifdef HAVE_READLINE
   build_completion_hash(1, 0);
@@ -4554,12 +4518,13 @@ com_rehash(String *buffer __attribute__((unused)),
 
 
 #ifdef USE_POPEN
-static int
+static int com_shell(String *, char *line)
-com_shell(String *buffer __attribute__((unused)),
-          char *line __attribute__((unused)))
 {
   char *shell_cmd;
 
+  if (status.sandbox)
+    return put_info("Not allowed in the sandbox mode", INFO_ERROR, 0);
+
   /* Skip space from line begin */
   while (my_isspace(charset_info, *line))
     line++;
@@ -4582,8 +4547,7 @@ com_shell(String *buffer __attribute__((unused)),
 #endif
 
 
-static int
+static int com_print(String *buffer,char *)
-com_print(String *buffer,char *line __attribute__((unused)))
 {
   tee_puts("--------------", stdout);
   (void) tee_fputs(buffer->c_ptr(), stdout);
@@ -4593,9 +4557,8 @@ com_print(String *buffer,char *line __attribute__((unused)))
   return 0;					/* If empty buffer */
 }
 
-	/* ARGSUSED */
+
-static int
+static int com_connect(String *buffer, char *line)
-com_connect(String *buffer, char *line)
 {
   char *tmp, buff[256];
   my_bool save_rehash= opt_rehash;
@@ -4648,8 +4611,7 @@ com_connect(String *buffer, char *line)
 }
 
 
-static int com_source(String *buffer __attribute__((unused)),
+static int com_source(String *, char *line)
-                      char *line)
 {
   char source_name[FN_REFLEN], *end, *param;
   LINE_BUFFER *line_buff;
@@ -4658,6 +4620,9 @@ static int com_source(String *buffer __attribute__((unused)),
   FILE *sql_file;
   my_bool save_ignore_errors;
 
+  if (status.sandbox)
+    return put_info("Not allowed in the sandbox mode", INFO_ERROR, 0);
+
   /* Skip space from file name */
   while (my_isspace(charset_info,*line))
     line++;
@@ -4692,6 +4657,7 @@ static int com_source(String *buffer __attribute__((unused)),
   bfill((char*) &status,sizeof(status),(char) 0);
 
   status.batch=old_status.batch;		// Run in batch mode
+  status.sandbox=old_status.sandbox;
   status.line_buff=line_buff;
   status.file_name=source_name;
   glob_buffer.length(0);			// Empty command buffer
@@ -4713,9 +4679,7 @@ static int com_source(String *buffer __attribute__((unused)),
 }
 
 
-	/* ARGSUSED */
+static int com_delimiter(String *, char *line)
-static int
-com_delimiter(String *buffer __attribute__((unused)), char *line)
 {
   char buff[256], *tmp;
 
@@ -4742,9 +4706,7 @@ com_delimiter(String *buffer __attribute__((unused)), char *line)
   return 0;
 }
 
-	/* ARGSUSED */
+static int com_use(String *, char *line)
-static int
-com_use(String *buffer __attribute__((unused)), char *line)
 {
   char *tmp, buff[FN_REFLEN + 1];
   int select_db;
@@ -4817,18 +4779,21 @@ com_use(String *buffer __attribute__((unused)), char *line)
   return 0;
 }
 
-static int
+static int com_sandbox(String *, char *)
-com_warnings(String *buffer __attribute__((unused)),
+{
-   char *line __attribute__((unused)))
+  status.sandbox= 1;
+  put_info("Sandbox mode.", INFO_INFO);
+  return 0;
+}
+
+static int com_warnings(String *, char *)
 {
   show_warnings = 1;
   put_info("Show warnings enabled.",INFO_INFO);
   return 0;
 }
 
-static int
+static int com_nowarnings(String *, char *)
-com_nowarnings(String *buffer __attribute__((unused)),
-   char *line __attribute__((unused)))
 {
   show_warnings = 0;
   put_info("Show warnings disabled.",INFO_INFO);
@@ -5092,10 +5057,7 @@ sql_connect(char *host,char *database,char *user,char *password,uint silent)
 }
 
 
-
+static int com_status(String *, char *)
-static int
-com_status(String *buffer __attribute__((unused)),
-	   char *line __attribute__((unused)))
 {
   const char *status_str;
   char buff[40];
@@ -5220,8 +5182,7 @@ select_limit, max_join_size);
   return 0;
 }
 
-static const char *
+static const char * server_version_string(MYSQL *con)
-server_version_string(MYSQL *con)
 {
   /* Only one thread calls this, so no synchronization is needed */
   if (server_version == NULL)
@@ -5346,8 +5307,7 @@ put_info(const char *str,INFO_TYPE info_type, uint error, const char *sqlstate)
 }
 
 
-static int
+static int put_error(MYSQL *con)
-put_error(MYSQL *con)
 {
   return put_info(mysql_error(con), INFO_ERROR, mysql_errno(con),
 		  mysql_sqlstate(con));
@@ -5694,8 +5654,7 @@ static void init_username()
   }
 }
 
-static int com_prompt(String *buffer __attribute__((unused)),
+static int com_prompt(String *, char *line)
-                      char *line)
 {
   char *ptr=strchr(line, ' ');
   prompt_counter = 0;

client/mysql_upgrade.c

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2006, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2017, MariaDB
+   Copyright (c) 2010, 2024, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -86,10 +86,10 @@ static struct my_option my_long_options[]=
   {"basedir", 'b',
    "Not used by mysql_upgrade. Only for backward compatibility.",
    0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"character-sets-dir", OPT_CHARSETS_DIR,
+  {"character-sets-dir", 0,
    "Not used by mysql_upgrade. Only for backward compatibility.",
    0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
-  {"compress", OPT_COMPRESS,
+  {"compress", 0,
    "Not used by mysql_upgrade. Only for backward compatibility.",
    &not_used, &not_used, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"datadir", 'd',
@@ -102,12 +102,12 @@ static struct my_option my_long_options[]=
   {"debug", '#', "Output debug log.",
    0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit.",
+  {"debug-check", 0, "Check memory and open file usage at exit.",
    &debug_check_flag, &debug_check_flag,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"debug-info", 'T', "Print some debug info at exit.", &debug_info_flag,
    &debug_info_flag, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"default-character-set", OPT_DEFAULT_CHARSET,
+  {"default-character-set", 0,
    "Not used by mysql_upgrade. Only for backward compatibility.",
    0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"default_auth", OPT_DEFAULT_AUTH,

client/mysqladmin.cc

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2014, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2019, MariaDB
+   Copyright (c) 2010, 2024, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -124,10 +124,10 @@ static struct my_option my_long_options[] =
   {"debug", '#', "Output debug log. Often this is 'd:t:o,filename'.",
    0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit.",
+  {"debug-check", 0, "Check memory and open file usage at exit.",
    &debug_check_flag, &debug_check_flag, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"debug-info", OPT_DEBUG_INFO, "Print some debug info at exit.",
+  {"debug-info", 0, "Print some debug info at exit.",
    &debug_info_flag, &debug_info_flag,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"force", 'f',
@@ -141,7 +141,7 @@ static struct my_option my_long_options[] =
   {"character-sets-dir", OPT_CHARSETS_DIR,
    "Directory for character set files.", &charsets_dir,
    &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"default-character-set", OPT_DEFAULT_CHARSET,
+  {"default-character-set", 0,
    "Set the default character set.", &default_charset,
    &default_charset, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"help", '?', "Display this help and exit.", 0, 0, 0, GET_NO_ARG,
@@ -195,21 +195,21 @@ static struct my_option my_long_options[] =
    NO_ARG, 0, 0, 0, 0, 0, 0},
   {"wait", 'w', "Wait and retry if connection is down.", 0, 0, 0, GET_UINT,
    OPT_ARG, 0, 0, 0, 0, 0, 0},
-  {"connect_timeout", OPT_CONNECT_TIMEOUT, "", &opt_connect_timeout,
+  {"connect_timeout", 0, "", &opt_connect_timeout,
    &opt_connect_timeout, 0, GET_ULONG, REQUIRED_ARG, 3600*12, 0,
    3600*12, 0, 1, 0},
-  {"shutdown_timeout", OPT_SHUTDOWN_TIMEOUT, "", &opt_shutdown_timeout,
+  {"shutdown_timeout", 0, "", &opt_shutdown_timeout,
    &opt_shutdown_timeout, 0, GET_ULONG, REQUIRED_ARG,
    SHUTDOWN_DEF_TIMEOUT, 0, 3600*12, 0, 1, 0},
-  {"wait_for_all_slaves", OPT_SHUTDOWN_WAIT_FOR_SLAVES,
+  {"wait_for_all_slaves", 0,
    "Defers shutdown until after all binlogged events have been sent to "
    "all connected slaves", &opt_shutdown_wait_for_slaves,
    &opt_shutdown_wait_for_slaves, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
-  {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
+  {"plugin_dir", 0, "Directory for client-side plugins.",
     &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"default_auth", OPT_DEFAULT_AUTH,
+  {"default_auth", 0,
    "Default authentication client-side plugin to use.",
    &opt_default_auth, &opt_default_auth, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -1342,7 +1342,9 @@ static void usage(void)
   refresh		Flush all tables and close and open logfiles\n\
   shutdown		Take server down\n\
   status		Gives a short status message from the server\n\
+  start-all-slaves	Start all slaves\n\
   start-slave		Start slave\n\
+  stop-all-slaves	Stop all slaves\n\
   stop-slave		Stop slave\n\
   variables             Prints variables available\n\
   version		Get version info from server");

client/mysqlbinlog.cc

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2014, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2020, MariaDB
+   Copyright (c) 2009, 2024, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1441,7 +1441,7 @@ static struct my_option my_options[] =
     like this:
     SET @`a`:=_cp850 0x4DFC6C6C6572 COLLATE `cp850_general_ci`;
   */
-  {"character-sets-dir", OPT_CHARSETS_DIR,
+  {"character-sets-dir", 0,
    "Directory for character set files.", &charsets_dir,
    &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"database", 'd', "List entries for just this database (local log only).",
@@ -1451,13 +1451,13 @@ static struct my_option my_options[] =
   {"debug", '#', "Output debug log.", &current_dbug_option,
    &current_dbug_option, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit .",
+  {"debug-check", 0, "Check memory and open file usage at exit .",
    &debug_check_flag, &debug_check_flag, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"debug-info", OPT_DEBUG_INFO, "Print some debug info at exit.",
+  {"debug-info", 0, "Print some debug info at exit.",
    &debug_info_flag, &debug_info_flag,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"default_auth", OPT_DEFAULT_AUTH,
+  {"default_auth", 0,
    "Default authentication client-side plugin to use.",
    &opt_default_auth, &opt_default_auth, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -1493,7 +1493,7 @@ static struct my_option my_options[] =
    0, GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"password", 'p', "Password to connect to remote server.",
    0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
-  {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
+  {"plugin_dir", 0, "Directory for client-side plugins.",
     &opt_plugindir, &opt_plugindir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"port", 'P', "Port number to use for connection or 0 for default to, in "
@@ -1519,14 +1519,14 @@ static struct my_option my_options[] =
    &result_file_name, &result_file_name, 0, GET_STR, REQUIRED_ARG,
    0, 0, 0, 0, 0, 0},
 #ifdef WHEN_FLASHBACK_REVIEW_READY
-  {"review", opt_flashback_review, "Print review sql in output file.",
+  {"review", 0, "Print review sql in output file.",
    &opt_flashback_review, &opt_flashback_review, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
    0, 0},
-  {"review-dbname", opt_flashback_flashback_review_dbname,
+  {"review-dbname", 0,
    "Writing flashback original row data into this db",
    &flashback_review_dbname, &flashback_review_dbname,
    0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"review-tablename", opt_flashback_flashback_review_tablename,
+  {"review-tablename", 0,
    "Writing flashback original row data into this table",
    &flashback_review_tablename, &flashback_review_tablename,
    0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -1576,7 +1576,7 @@ static struct my_option my_options[] =
    "Alias for --do-server-ids.",
    &server_id_str, &server_id_str, 0, GET_STR_ALLOC,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"set-charset", OPT_SET_CHARSET,
+  {"set-charset", 0,
    "Add 'SET NAMES character_set' to the output.", &charset,
    &charset, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"short-form", 's', "Just show regular queries: no extra info, no "
@@ -1659,7 +1659,7 @@ that may lead to an endless loop.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"version", 'V', "Print version and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0,
    0, 0, 0, 0, 0},
-  {"open_files_limit", OPT_OPEN_FILES_LIMIT,
+  {"open_files_limit", 0,
    "Used to reserve file descriptors for use by this program.",
    &open_files_limit, &open_files_limit, 0, GET_ULONG,
    REQUIRED_ARG, MY_NFILE, 8, OS_FILE_LIMIT, 0, 1, 0},
@@ -1685,12 +1685,12 @@ that may lead to an endless loop.",
    "Updates to a database with a different name than the original. \
 Example: rewrite-db='from->to'.",
    0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"skip-annotate-row-events", OPT_SKIP_ANNOTATE_ROWS_EVENTS,
+  {"skip-annotate-row-events", 0,
    "Don't print Annotate_rows events stored in the binary log.",
    (uchar**) &opt_skip_annotate_row_events,
    (uchar**) &opt_skip_annotate_row_events,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"print-table-metadata", OPT_PRINT_TABLE_METADATA,
+  {"print-table-metadata", 0,
    "Print metadata stored in Table_map_log_event",
    &opt_print_table_metadata, &opt_print_table_metadata, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
@@ -2115,11 +2115,6 @@ get_one_option(const struct my_option *opt, const char *argument,
       die(1);
     }
     break;
-#ifdef WHEN_FLASHBACK_REVIEW_READY
-  case opt_flashback_review:
-    opt_flashback_review= 1;
-    break;
-#endif
   case OPT_START_DATETIME:
     start_datetime= convert_str_to_timestamp(start_datetime_str);
     break;
@@ -3232,7 +3227,8 @@ int main(int argc, char** argv)
   {
     if (!opt_version)
     {
-      usage();
+      error("Please provide the log file(s). Run with '--help' for usage "
+            "instructions.");
       retval= ERROR_STOP;
     }
     goto err;

client/mysqlcheck.c

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2001, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2012, MariaDB
+   Copyright (c) 2010, 2024, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -82,11 +82,11 @@ static struct my_option my_long_options[] =
    "Instead of issuing one query for each table, use one query per database, naming all tables in the database in a comma-separated list.",
    &opt_all_in_1, &opt_all_in_1, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
-  {"auto-repair", OPT_AUTO_REPAIR,
+  {"auto-repair", 0,
    "If a checked table is corrupted, automatically fix it. Repairing will be done after all tables have been checked, if corrupted ones were found.",
    &opt_auto_repair, &opt_auto_repair, 0, GET_BOOL, NO_ARG, 0,
    0, 0, 0, 0, 0},
-  {"character-sets-dir", OPT_CHARSETS_DIR,
+  {"character-sets-dir", 0,
    "Directory for character set files.", (char**) &charsets_dir,
    (char**) &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"check", 'c', "Check table for errors.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0,
@@ -97,7 +97,7 @@ static struct my_option my_long_options[] =
   {"check-upgrade", 'g',
    "Check tables for version-dependent changes. May be used with --auto-repair to correct tables requiring version-dependent updates.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"compress", OPT_COMPRESS, "Use compression in server/client protocol.",
+  {"compress", 0, "Use compression in server/client protocol.",
    &opt_compress, &opt_compress, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
   {"databases", 'B',
@@ -111,16 +111,16 @@ static struct my_option my_long_options[] =
   {"debug", '#', "Output debug log. Often this is 'd:t:o,filename'.",
    0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit.",
+  {"debug-check", 0, "Check memory and open file usage at exit.",
    &debug_check_flag, &debug_check_flag, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"debug-info", OPT_DEBUG_INFO, "Print some debug info at exit.",
+  {"debug-info", 0, "Print some debug info at exit.",
    &debug_info_flag, &debug_info_flag,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"default-character-set", OPT_DEFAULT_CHARSET,
+  {"default-character-set", 0,
    "Set the default character set.", &default_charset,
    &default_charset, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"default_auth", OPT_DEFAULT_AUTH,
+  {"default_auth", 0,
    "Default authentication client-side plugin to use.",
    &opt_default_auth, &opt_default_auth, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -140,7 +140,7 @@ static struct my_option my_long_options[] =
    "If you are using this option with CHECK TABLE, it will ensure that the table is 100 percent consistent, but will take a long time. If you are using this option with REPAIR TABLE, it will force using old slow repair with keycache method, instead of much faster repair by sorting.",
    &opt_extended, &opt_extended, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
-  {"flush", OPT_FLUSH_TABLES, "Flush each table after check. This is useful if you don't want to have the checked tables take up space in the caches after the check",
+  {"flush", 0, "Flush each table after check. This is useful if you don't want to have the checked tables take up space in the caches after the check",
    &opt_flush_tables, &opt_flush_tables, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
    0, 0 },
   {"help", '?', "Display this help message and exit.", 0, 0, 0, GET_NO_ARG,
@@ -150,7 +150,7 @@ static struct my_option my_long_options[] =
   {"medium-check", 'm',
    "Faster than extended-check, but only finds 99.99 percent of all errors. Should be good enough for most cases.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"write-binlog", OPT_WRITE_BINLOG,
+  {"write-binlog", 0,
    "Log ANALYZE, OPTIMIZE and REPAIR TABLE commands. Use --skip-write-binlog "
    "when commands should not be sent to replication slaves.",
    &opt_write_binlog, &opt_write_binlog, 0, GET_BOOL, NO_ARG,
@@ -168,7 +168,7 @@ static struct my_option my_long_options[] =
   {"pipe", 'W', "Use named pipes to connect to server.", 0, 0, 0, GET_NO_ARG,
    NO_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
+  {"plugin_dir", 0, "Directory for client-side plugins.",
    &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"port", 'P', "Port number to use for connection or 0 for default to, in "
@@ -199,7 +199,7 @@ static struct my_option my_long_options[] =
 #include <sslopt-longopts.h>
   {"tables", OPT_TABLES, "Overrides option --databases (-B).", 0, 0, 0,
    GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"use-frm", OPT_FRM,
+  {"use-frm", 0,
    "When used with REPAIR, get table structure from .frm file, so the table can be repaired even if .MYI header is corrupted.",
    &opt_frm, &opt_frm, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0,
    0},
@@ -895,6 +895,7 @@ static int disable_binlog()
   return run_query("SET SQL_LOG_BIN=0", 0);
 }
 
+
 static int handle_request_for_tables(char *tables, size_t length,
                                      my_bool view, my_bool dont_quote)
 {
@@ -1026,7 +1027,10 @@ static void insert_table_name(DYNAMIC_ARRAY *arr, char *in, size_t dblen)
   insert_dynamic(arr, (uchar*) buf);
 }
 
-static void print_result()
+/* Ok as mysqlcheck is not multi threaded */
+PRAGMA_DISABLE_CHECK_STACK_FRAME
+
+static void __attribute__((noinline)) print_result()
 {
   MYSQL_RES *res;
   MYSQL_ROW row;
@@ -1117,6 +1121,7 @@ static void print_result()
   mysql_free_result(res);
   DBUG_VOID_RETURN;
 }
+PRAGMA_REENABLE_CHECK_STACK_FRAME
 
 
 static int dbConnect(char *host, char *user, char *passwd)

client/mysqldump.cc

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2020, MariaDB Corporation.
+   Copyright (c) 2010, 2024, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -166,7 +166,6 @@ static my_bool server_supports_switching_charsets= TRUE;
 static ulong opt_compatible_mode= 0;
 #define MYSQL_OPT_MASTER_DATA_EFFECTIVE_SQL 1
 #define MYSQL_OPT_MASTER_DATA_COMMENTED_SQL 2
-#define MYSQL_OPT_MAX_STATEMENT_TIME 0
 #define MYSQL_OPT_SLAVE_DATA_EFFECTIVE_SQL 1
 #define MYSQL_OPT_SLAVE_DATA_COMMENTED_SQL 2
 static uint opt_mysql_port= 0, opt_master_data;
@@ -265,41 +264,36 @@ static struct my_option my_long_options[] =
 {
   {"all-databases", 'A',
    "Dump all the databases. This will be same as --databases with all databases selected.",
-   &opt_alldbs, &opt_alldbs, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+   &opt_alldbs, &opt_alldbs, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0},
   {"all-tablespaces", 'Y',
    "Dump all the tablespaces.",
-   &opt_alltspcs, &opt_alltspcs, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+   &opt_alltspcs, &opt_alltspcs, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0},
   {"no-tablespaces", 'y',
    "Do not dump any tablespace information.",
-   &opt_notspcs, &opt_notspcs, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+   &opt_notspcs, &opt_notspcs, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0},
+  {"add-drop-database", 0, "Add a DROP DATABASE before each create.",
-  {"add-drop-database", OPT_DROP_DATABASE, "Add a DROP DATABASE before each create.",
    &opt_drop_database, &opt_drop_database, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0,
    0},
-  {"add-drop-table", OPT_DROP, "Add a DROP TABLE before each create.",
+  {"add-drop-table", 0, "Add a DROP TABLE before each create.",
-   &opt_drop, &opt_drop, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0,
+   &opt_drop, &opt_drop, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-   0},
   {"add-drop-trigger", 0, "Add a DROP TRIGGER before each create.",
    &opt_drop_trigger, &opt_drop_trigger, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0,
    0},
-  {"add-locks", OPT_LOCKS, "Add locks around INSERT statements.",
+  {"add-locks", 0, "Add locks around INSERT statements.",
-   &opt_lock, &opt_lock, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0,
+   &opt_lock, &opt_lock, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-   0},
+  {"allow-keywords", 0,
-  {"allow-keywords", OPT_KEYWORDS,
    "Allow creation of column names that are keywords.", &opt_keywords,
    &opt_keywords, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"apply-slave-statements", OPT_MYSQLDUMP_SLAVE_APPLY,
+  {"apply-slave-statements", 0,
    "Adds 'STOP SLAVE' prior to 'CHANGE MASTER' and 'START SLAVE' to bottom of dump.",
-   &opt_slave_apply, &opt_slave_apply, 0, GET_BOOL, NO_ARG,
+   &opt_slave_apply, &opt_slave_apply, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0, 0, 0},
+  {"as-of", 0,
-  {"as-of", OPT_ASOF_TIMESTAMP,
    "Dump system versioned table(s) as of specified timestamp. "
    "Argument is interpreted according to the --tz-utc setting. "
    "Table structures are always dumped as of current timestamp.",
-   &opt_asof_timestamp, &opt_asof_timestamp, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+   &opt_asof_timestamp, &opt_asof_timestamp, 0, GET_STR, REQUIRED_ARG,
-  {"character-sets-dir", OPT_CHARSETS_DIR,
+   0, 0, 0, 0, 0, 0},
+  {"character-sets-dir", 0,
    "Directory for character set files.", (char **)&charsets_dir,
    (char **)&charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"comments", 'i', "Write additional information.",
@@ -324,21 +318,18 @@ static struct my_option my_long_options[] =
    &opt_complete_insert, &opt_complete_insert, 0, GET_BOOL,
    NO_ARG, 0, 0, 0, 0, 0, 0},
   {"compress", 'C', "Use compression in server/client protocol.",
-   &opt_compress, &opt_compress, 0, GET_BOOL, NO_ARG, 0, 0, 0,
+   &opt_compress, &opt_compress, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0},
+  {"copy_s3_tables", 0,
-  {"copy_s3_tables", OPT_COPY_S3_TABLES,
    "If 'no' S3 tables will be ignored, otherwise S3 tables will be copied as "
    " Aria tables and then altered to S3",
    &opt_copy_s3_tables, &opt_copy_s3_tables, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
   {"create-options", 'a',
    "Include all MariaDB specific create options.",
-   &create_options, &create_options, 0, GET_BOOL, NO_ARG, 1,
+   &create_options, &create_options, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-   0, 0, 0, 0, 0},
   {"databases", 'B',
    "Dump several databases. Note the difference in usage; in this case no tables are given. All name arguments are regarded as database names. 'USE db_name;' will be included in the output.",
-   &opt_databases, &opt_databases, 0, GET_BOOL, NO_ARG, 0, 0,
+   &opt_databases, &opt_databases, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0},
 #ifdef DBUG_OFF
   {"debug", '#', "This is a non-debug version. Catch this and exit.",
    0,0, 0, GET_DISABLED, OPT_ARG, 0, 0, 0, 0, 0, 0},
@@ -346,19 +337,17 @@ static struct my_option my_long_options[] =
   {"debug", '#', "Output debug log.", (char *)&default_dbug_option,
    (char *)&default_dbug_option, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit.",
+  {"debug-check", 0, "Check memory and open file usage at exit.",
    &debug_check_flag, &debug_check_flag, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"debug-info", OPT_DEBUG_INFO, "Print some debug info at exit.",
+  {"debug-info", 0, "Print some debug info at exit.", &debug_info_flag,
-   &debug_info_flag, &debug_info_flag,
+    &debug_info_flag, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"default-character-set", OPT_DEFAULT_CHARSET,
    "Set the default character set.", &default_charset,
    &default_charset, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"delayed-insert", OPT_DELAYED, "Insert rows with INSERT DELAYED.",
+  {"delayed-insert", 0, "Insert rows with INSERT DELAYED.",
-   &opt_delayed, &opt_delayed, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+   &opt_delayed, &opt_delayed, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0},
+  {"delete-master-logs", 0,
-  {"delete-master-logs", OPT_DELETE_MASTER_LOGS,
    "Delete logs on master after backup. This automatically enables --master-data.",
    &opt_delete_master_logs, &opt_delete_master_logs, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
@@ -372,9 +361,8 @@ static struct my_option my_long_options[] =
    "'/*!40000 ALTER TABLE tb_name DISABLE KEYS */; and '/*!40000 ALTER "
    "TABLE tb_name ENABLE KEYS */; will be put in the output.", &opt_disable_keys,
    &opt_disable_keys, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-  {"dump-date", OPT_DUMP_DATE, "Put a dump date to the end of the output.",
+  {"dump-date", 0, "Put a dump date to the end of the output.",
-   &opt_dump_date, &opt_dump_date, 0,
+   &opt_dump_date, &opt_dump_date, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-   GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
   {"dump-history", 'H', "Dump system-versioned tables with history (only for "
     "timestamp based versioning)", &opt_dump_history,
     &opt_dump_history, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
@@ -391,24 +379,23 @@ static struct my_option my_long_options[] =
    "Option automatically turns --lock-tables off.",
    &opt_slave_data, &opt_slave_data, 0,
    GET_UINT, OPT_ARG, 0, 0, MYSQL_OPT_SLAVE_DATA_COMMENTED_SQL, 0, 0, 0},
-  {"events", 'E', "Dump events.",
+  {"events", 'E', "Dump events.", &opt_events, &opt_events, 0, GET_BOOL,
-     &opt_events, &opt_events, 0, GET_BOOL,
     NO_ARG, 0, 0, 0, 0, 0, 0},
   {"extended-insert", 'e',
    "Use multiple-row INSERT syntax that include several VALUES lists.",
    &extended_insert, &extended_insert, 0, GET_BOOL, NO_ARG,
    1, 0, 0, 0, 0, 0},
-  {"fields-terminated-by", OPT_FTB,
+  {"fields-terminated-by", 0,
    "Fields in the output file are terminated by the given string.",
    &fields_terminated, &fields_terminated, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"fields-enclosed-by", OPT_ENC,
+  {"fields-enclosed-by", 0,
    "Fields in the output file are enclosed by the given character.",
    &enclosed, &enclosed, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0 ,0, 0},
-  {"fields-optionally-enclosed-by", OPT_O_ENC,
+  {"fields-optionally-enclosed-by", 0,
    "Fields in the output file are optionally enclosed by the given character.",
    &opt_enclosed, &opt_enclosed, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0 ,0, 0},
-  {"fields-escaped-by", OPT_ESC,
+  {"fields-escaped-by", 0,
    "Fields in the output file are escaped by the given character.",
    &escaped, &escaped, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"flush-logs", 'F', "Flush logs file in server before starting dump. "
@@ -420,29 +407,26 @@ static struct my_option my_long_options[] =
    "to the moment all tables are locked. So if you want your dump and "
    "the log flush to happen at the same exact moment you should use "
    "--lock-all-tables or --master-data with --flush-logs.",
-   &flush_logs, &flush_logs, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+   &flush_logs, &flush_logs, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0},
+  {"flush-privileges", 0, "Emit a FLUSH PRIVILEGES statement "
-  {"flush-privileges", OPT_ESC, "Emit a FLUSH PRIVILEGES statement "
    "after dumping the mysql database.  This option should be used any "
    "time the dump contains the mysql database and any other database "
    "that depends on the data in the mysql database for proper restore. ",
    &flush_privileges, &flush_privileges, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
    0, 0},
   {"force", 'f', "Continue even if we get an SQL error.",
-   &ignore_errors, &ignore_errors, 0, GET_BOOL, NO_ARG,
+   &ignore_errors, &ignore_errors, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0, 0, 0},
   {"gtid", 0, "Used together with --master-data=1 or --dump-slave=1."
    "When enabled, the output from those options will set the GTID position "
    "instead of the binlog file and offset; the file/offset will appear only as "
    "a comment. When disabled, the GTID position will still appear in the "
    "output, but only commented.",
-   &opt_use_gtid, &opt_use_gtid, 0, GET_BOOL, NO_ARG,
+   &opt_use_gtid, &opt_use_gtid, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0, 0, 0},
   {"header", 0, "Used together with --tab. When enabled, adds header with column names to the top of output txt files.",
    &opt_header, &opt_header, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"help", '?', "Display this help message and exit.", 0, 0, 0, GET_NO_ARG,
    NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"hex-blob", OPT_HEXBLOB, "Dump binary strings (BINARY, "
+  {"hex-blob", 0, "Dump binary strings (BINARY, "
     "VARBINARY, BLOB) in hexadecimal format.",
    &opt_hex_blob, &opt_hex_blob, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"host", 'h', "Connect to host.", &current_host,
@@ -464,15 +448,13 @@ static struct my_option my_long_options[] =
    "be specified with both database and table names, e.g., "
    "--ignore-table=database.table.",
    0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"include-master-host-port", OPT_MYSQLDUMP_INCLUDE_MASTER_HOST_PORT,
+  {"include-master-host-port", 0,
    "Adds 'MASTER_HOST=<host>, MASTER_PORT=<port>' to 'CHANGE MASTER TO..' "
    "in dump produced with --dump-slave.", &opt_include_master_host_port,
-   &opt_include_master_host_port, 0, GET_BOOL, NO_ARG,
+   &opt_include_master_host_port, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0, 0, 0},
+  {"insert-ignore", 0, "Insert rows with INSERT IGNORE.",
-  {"insert-ignore", OPT_INSERT_IGNORE, "Insert rows with INSERT IGNORE.",
+   &opt_ignore, &opt_ignore, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   &opt_ignore, &opt_ignore, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+  {"lines-terminated-by", 0,
-   0, 0},
-  {"lines-terminated-by", OPT_LTB,
    "Lines in the output file are terminated by the given string.",
    &lines_terminated, &lines_terminated, 0, GET_STR,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -483,7 +465,7 @@ static struct my_option my_long_options[] =
    0, 0, 0, 0, 0, 0},
   {"lock-tables", 'l', "Lock all tables for read.", &lock_tables,
    &lock_tables, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-  {"log-error", OPT_ERROR_LOG_FILE, "Append warnings and errors to given file.",
+  {"log-error", 0, "Append warnings and errors to given file.",
    &log_error_file, &log_error_file, 0, GET_STR,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"log-queries", 0, "When restoring the dump, the server will, if logging turned on, log the queries to the general and slow query log.",
@@ -500,30 +482,28 @@ static struct my_option my_long_options[] =
    "Option automatically turns --lock-tables off.",
    &opt_master_data, &opt_master_data, 0,
    GET_UINT, OPT_ARG, 0, 0, MYSQL_OPT_MASTER_DATA_COMMENTED_SQL, 0, 0, 0},
-  {"max_allowed_packet", OPT_MAX_ALLOWED_PACKET, 
+  {"max_allowed_packet", 0,
    "The maximum packet length to send to or receive from server.",
     &opt_max_allowed_packet, &opt_max_allowed_packet, 0,
     GET_ULONG, REQUIRED_ARG, 24*1024*1024, 4096,
    (longlong) 2L*1024L*1024L*1024L, MALLOC_OVERHEAD, 1024, 0},
-  {"max-statement-time", MYSQL_OPT_MAX_STATEMENT_TIME,
+  {"max-statement-time", 0,
    "Max statement execution time. If unset, overrides server default with 0.",
    &opt_max_statement_time, &opt_max_statement_time, 0, GET_DOUBLE,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"net_buffer_length", OPT_NET_BUFFER_LENGTH, 
+  {"net_buffer_length", 0,
    "The buffer size for TCP/IP and socket communication.",
     &opt_net_buffer_length, &opt_net_buffer_length, 0,
     GET_ULONG, REQUIRED_ARG, 1024*1024L-1025, 4096, 16*1024L*1024L,
    MALLOC_OVERHEAD-1024, 1024, 0},
-  {"no-autocommit", OPT_AUTOCOMMIT,
+  {"no-autocommit", 0,
    "Wrap tables with autocommit/commit statements.",
-   &opt_autocommit, &opt_autocommit, 0, GET_BOOL, NO_ARG,
+   &opt_autocommit, &opt_autocommit, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0, 0, 0},
   {"no-create-db", 'n',
    "Suppress the CREATE DATABASE ... IF EXISTS statement that normally is "
    "output for each dumped database if --all-databases or --databases is "
    "given.",
-   &opt_create_db, &opt_create_db, 0, 
+   &opt_create_db, &opt_create_db, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"no-create-info", 't', "Don't write table creation info.",
    &opt_no_create_info, &opt_no_create_info, 0, GET_BOOL,
    NO_ARG, 0, 0, 0, 0, 0, 0},
@@ -537,7 +517,7 @@ static struct my_option my_long_options[] =
   {"opt", OPT_OPTIMIZE,
    "Same as --add-drop-table, --add-locks, --create-options, --quick, --extended-insert, --lock-tables, --set-charset, and --disable-keys. Enabled by default, disable with --skip-opt.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"order-by-primary", OPT_ORDER_BY_PRIMARY,
+  {"order-by-primary", 0,
    "Sorts each table's rows by primary key, or first unique key, if such a key exists.  Useful when dumping a MyISAM table to be loaded into an InnoDB table, but will make the dump itself take considerably longer.",
    &opt_order_by_primary, &opt_order_by_primary, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"order-by-size", 0,
@@ -564,27 +544,24 @@ static struct my_option my_long_options[] =
   {"quote-names",'Q', "Quote table and column names with backticks (`).",
    &opt_quoted, &opt_quoted, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0,
    0, 0},
-  {"replace", OPT_MYSQL_REPLACE_INTO, "Use REPLACE INTO instead of INSERT INTO.",
+  {"replace", 0, "Use REPLACE INTO instead of INSERT INTO.", &opt_replace_into,
-   &opt_replace_into, &opt_replace_into, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+    &opt_replace_into, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0},
   {"result-file", 'r',
    "Direct output to a given file. This option should be used in systems "
    "(e.g., DOS, Windows) that use carriage-return linefeed pairs (\\r\\n) "
    "to separate text lines. This option ensures that only a single newline "
    "is used.", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"routines", 'R', "Dump stored routines (functions and procedures).",
-   &opt_routines, &opt_routines, 0, GET_BOOL,
+   &opt_routines, &opt_routines, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-   NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"set-charset", 0,
-  {"set-charset", OPT_SET_CHARSET,
+   "Add 'SET NAMES default_character_set' to the output.", &opt_set_charset,
-   "Add 'SET NAMES default_character_set' to the output.",
+   &opt_set_charset, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-   &opt_set_charset, &opt_set_charset, 0, GET_BOOL, NO_ARG, 1,
-   0, 0, 0, 0, 0},
   /*
     Note that the combination --single-transaction --master-data
     will give bullet-proof binlog position only if server >=4.1.3. That's the
     old "FLUSH TABLES WITH READ LOCK does not block commit" fixed bug.
   */
-  {"single-transaction", OPT_TRANSACTION,
+  {"single-transaction", 0,
    "Creates a consistent snapshot by dumping all tables in a single "
    "transaction. Works ONLY for tables stored in storage engines which "
    "support multiversioning (currently only InnoDB does); the dump is NOT "
@@ -603,7 +580,7 @@ static struct my_option my_long_options[] =
    &opt_mysql_unix_port, &opt_mysql_unix_port, 0, 
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 #include <sslopt-longopts.h>
-  {"system", 256, "Dump system tables as portable SQL",
+  {"system", 0, "Dump system tables as portable SQL",
    &opt_system, &opt_system, &opt_system_types, GET_SET, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"tab",'T',
    "Create tab-separated textfile for each table to given path. (Create .sql "
@@ -612,17 +589,16 @@ static struct my_option my_long_options[] =
    &path, &path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"tables", OPT_TABLES, "Overrides option --databases (-B).",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"triggers", OPT_TRIGGERS, "Dump triggers for each dumped table.",
+  {"triggers", 0, "Dump triggers for each dumped table.",
    &opt_dump_triggers, &opt_dump_triggers, 0, GET_BOOL,
    NO_ARG, 1, 0, 0, 0, 0, 0},
-  {"tz-utc", OPT_TZ_UTC,
+  {"tz-utc", 0,
    "Set connection time zone to UTC before commencing the dump and add "
    "SET TIME_ZONE=´+00:00´ to the top of the dump file.",
     &opt_tz_utc, &opt_tz_utc, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
 #ifndef DONT_ALLOW_USER_CHANGE
-  {"user", 'u', "User for login if not current user.",
+  {"user", 'u', "User for login if not current user.", &current_user,
-   &current_user, &current_user, 0, GET_STR, REQUIRED_ARG,
+    &current_user, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0, 0, 0},
 #endif
   {"verbose", 'v', "Print info about the various stages.",
    &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
@@ -632,11 +608,10 @@ static struct my_option my_long_options[] =
    &where, &where, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"xml", 'X', "Dump a database as well formed XML.", 0, 0, 0, GET_NO_ARG,
    NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
+  {"plugin_dir", 0, "Directory for client-side plugins.",
    &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"default_auth", OPT_DEFAULT_AUTH,
+  {"default_auth", 0, "Default authentication client-side plugin to use.",
-   "Default authentication client-side plugin to use.",
    &opt_default_auth, &opt_default_auth, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
@@ -799,12 +774,13 @@ static void write_header(FILE *sql_file, const char *db_name)
     fputs(">\n", sql_file);
     check_io(sql_file);
   }
-  else if (!opt_compact)
+  else
   {
-    print_comment(sql_file, 0,
+    fprintf(sql_file, "/*!999999\\- enable the sandbox mode */ \n");
-                  "-- MariaDB dump %s-%s, for %s (%s)\n--\n",
+    if (!opt_compact)
-                  VER, MYSQL_SERVER_VERSION, SYSTEM_TYPE,
+    {
-                  MACHINE_TYPE);
+      print_comment(sql_file, 0, "-- MariaDB dump %s-%s, for %s (%s)\n--\n",
+                    VER, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE);
       print_comment(sql_file, 0, "-- Host: %s    ",
                     fix_for_comment(current_host ? current_host : "localhost"));
       print_comment(sql_file, 0, "Database: %s\n",
@@ -832,23 +808,24 @@ static void write_header(FILE *sql_file, const char *db_name)
         fprintf(sql_file, "/*!40103 SET TIME_ZONE='+00:00' */;\n");
       }
 
+      if (!path)
+      {
         if (!multi_file_output)
-    {
-      if (!opt_no_create_info)
         {
           /* We don't need unique checks as the table is created just before */
-        fprintf(md_result_file,"\
+          fprintf(md_result_file,
-/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;\n");
+            "/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;\n");
         }
-      fprintf(md_result_file,"\
+        fprintf(md_result_file,
-/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;\n\
+          "/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;\n");
-");
       }
       fprintf(sql_file,
               "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='%s%s%s' */;\n"
               "/*M!100616 SET @OLD_NOTE_VERBOSITY=@@NOTE_VERBOSITY, NOTE_VERBOSITY=0 */;\n",
-            multi_file_output?"":"NO_AUTO_VALUE_ON_ZERO",compatible_mode_normal_str[0]==0?"":",",
+              multi_file_output?"":"NO_AUTO_VALUE_ON_ZERO",
+              compatible_mode_normal_str[0]==0?"":",",
               compatible_mode_normal_str);
+    }
     check_io(sql_file);
   }
 } /* write_header */
@@ -1327,8 +1304,9 @@ static int get_options(int *argc, char ***argv)
   if (opt_slave_data)
   {
     opt_lock_all_tables= !opt_single_transaction;
-    opt_master_data= 0;
     opt_delete_master_logs= 0;
+    if (opt_slave_data != MYSQL_OPT_SLAVE_DATA_COMMENTED_SQL)
+      opt_master_data= 0;
   }
 
   /* Ensure consistency of the set of binlog & locking options */
@@ -1341,10 +1319,7 @@ static int get_options(int *argc, char ***argv)
     return(EX_USAGE);
   }
   if (opt_master_data)
-  {
     opt_lock_all_tables= !opt_single_transaction;
-    opt_slave_data= 0;
-  }
   if (opt_single_transaction || opt_lock_all_tables)
     lock_tables= 0;
   if (enclosed && opt_enclosed)
@@ -3201,6 +3176,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
     if (opt_header)
       dynstr_set_checked(&select_field_names_for_header, "");
   }
+
   insert_option= ((delayed && opt_ignore) ? "DELAYED IGNORE " :
                   delayed ? "DELAYED " : opt_ignore ? "IGNORE " : "");
 
@@ -6344,17 +6320,12 @@ static int do_show_master_status(MYSQL *mysql_con, int consistent_binlog_pos,
 
   }
 
-  /* SHOW MASTER STATUS reports file and position */
+  /* gtid */
-  print_comment(md_result_file, 0,
-                "\n--\n-- Position to start replication or point-in-time "
-                "recovery from\n--\n\n");
-  fprintf(md_result_file,
-          "%sCHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%s;\n",
-          (use_gtid ? "-- " : comment_prefix), file, offset);
   if (have_mariadb_gtid)
   {
     print_comment(md_result_file, 0,
-                  "\n--\n-- GTID to start replication from\n--\n\n");
+                  "\n-- Preferably use GTID to start replication from GTID "
+                  "position:\n\n");
     if (use_gtid)
       fprintf(md_result_file,
               "%sCHANGE MASTER TO MASTER_USE_GTID=slave_pos;\n",
@@ -6363,6 +6334,19 @@ static int do_show_master_status(MYSQL *mysql_con, int consistent_binlog_pos,
             "%sSET GLOBAL gtid_slave_pos='%s';\n",
             (!use_gtid ? "-- " : comment_prefix), gtid_pos);
   }
+
+  /* SHOW MASTER STATUS reports file and position */
+  print_comment(md_result_file, 0,
+                "\n--\n-- Alternately, following is the position of the binary "
+                "logging from SHOW MASTER STATUS at point of backup."
+                "\n-- Use this when creating a replica of the primary server "
+                "where the backup was made."
+                "\n-- The new server will be connecting to the primary server "
+                "where the backup was taken."
+                "\n--\n\n");
+  fprintf(md_result_file,
+          "%sCHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%s;\n",
+          (use_gtid ? "-- " : comment_prefix), file, offset);
   check_io(md_result_file);
 
   if (!consistent_binlog_pos)
@@ -6441,7 +6425,6 @@ static int do_show_slave_status(MYSQL *mysql_con, int use_gtid,
     (opt_slave_data == MYSQL_OPT_SLAVE_DATA_COMMENTED_SQL) ? "-- " : "";
   const char *gtid_comment_prefix= (use_gtid ? comment_prefix : "-- ");
   const char *nogtid_comment_prefix= (!use_gtid ? comment_prefix : "-- ");
-  int set_gtid_done= 0;
 
   if (mysql_query_with_error_report(mysql_con, &slave,
                                     multi_source ?
@@ -6457,9 +6440,16 @@ static int do_show_slave_status(MYSQL *mysql_con, int use_gtid,
     return 1;
   }
 
-  while ((row= mysql_fetch_row(slave)))
+  print_comment(md_result_file, 0,
-  {
+                "\n--\n-- The following is the SQL position of the replication "
-    if (multi_source && !set_gtid_done)
+                "taken from SHOW SLAVE STATUS at the time of backup.\n"
+                "-- Use this position when creating a clone of, or replacement "
+                "server, from where the backup was taken."
+                "\n-- This new server will connects to the same primary "
+                "server%s.\n--\n",
+                multi_source ? "(s)" : "");
+
+  if (multi_source)
   {
     char gtid_pos[MAX_GTID_LENGTH];
     if (have_mariadb_gtid && get_gtid_pos(gtid_pos, 0))
@@ -6467,13 +6457,19 @@ static int do_show_slave_status(MYSQL *mysql_con, int use_gtid,
       mysql_free_result(slave);
       return 1;
     }
-      if (opt_comments)
+    print_comment(md_result_file, 0,
-        fprintf(md_result_file, "\n--\n-- Gtid position to start replication "
+                  "-- GTID position to start replication:\n");
-                "from\n--\n\n");
     fprintf(md_result_file, "%sSET GLOBAL gtid_slave_pos='%s';\n",
             gtid_comment_prefix, gtid_pos);
-      set_gtid_done= 1;
   }
+  if (use_gtid)
+    print_comment(md_result_file, 0,
+                  "\n-- Use only the MASTER_USE_GTID=slave_pos or "
+                  "MASTER_LOG_FILE/MASTER_LOG_POS in the statements below."
+                  "\n\n");
+
+  while ((row= mysql_fetch_row(slave)))
+  {
     if (row[9 + multi_source] && row[21 + multi_source])
     {
       if (use_gtid)
@@ -6487,11 +6483,6 @@ static int do_show_slave_status(MYSQL *mysql_con, int use_gtid,
       }
 
       /* SHOW MASTER STATUS reports file and position */
-      if (opt_comments)
-        fprintf(md_result_file,
-                "\n--\n-- Position to start replication or point-in-time "
-                "recovery from (the master of this slave)\n--\n\n");
-
       if (multi_source)
         fprintf(md_result_file, "%sCHANGE MASTER '%.80s' TO ",
                 nogtid_comment_prefix, row[0]);
@@ -6512,6 +6503,7 @@ static int do_show_slave_status(MYSQL *mysql_con, int use_gtid,
       check_io(md_result_file);
     }
   }
+  fprintf(md_result_file, "\n");
   mysql_free_result(slave);
   return 0;
 }

client/mysqlimport.c

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2015, Oracle and/or its affiliates.
-   Copyright (c) 2011, 2022, MariaDB
+   Copyright (c) 2011, 2024, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -70,10 +70,10 @@ static char **argv_to_free;
 
 static struct my_option my_long_options[] =
 {
-  {"character-sets-dir", OPT_CHARSETS_DIR,
+  {"character-sets-dir", 0,
    "Directory for character set files.", (char**) &charsets_dir,
    (char**) &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"default-character-set", OPT_DEFAULT_CHARSET,
+  {"default-character-set", 0,
    "Set the default character set.", &default_charset,
    &default_charset, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"columns", 'c',
@@ -85,31 +85,31 @@ static struct my_option my_long_options[] =
    0, 0, 0},
   {"debug",'#', "Output debug log. Often this is 'd:t:o,filename'.", 0, 0, 0,
    GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit.",
+  {"debug-check", 0, "Check memory and open file usage at exit.",
    &debug_check_flag, &debug_check_flag, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"debug-info", OPT_DEBUG_INFO, "Print some debug info at exit.",
+  {"debug-info", 0, "Print some debug info at exit.",
    &debug_info_flag, &debug_info_flag,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"default_auth", OPT_DEFAULT_AUTH,
+  {"default_auth", 0,
    "Default authentication client-side plugin to use.",
    &opt_default_auth, &opt_default_auth, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"delete", 'd', "First delete all rows from table.", &opt_delete,
    &opt_delete, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"fields-terminated-by", OPT_FTB,
+  {"fields-terminated-by", 0,
    "Fields in the input file are terminated by the given string.", 
    &fields_terminated, &fields_terminated, 0, 
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"fields-enclosed-by", OPT_ENC,
+  {"fields-enclosed-by", 0,
    "Fields in the import file are enclosed by the given character.", 
    &enclosed, &enclosed, 0, 
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"fields-optionally-enclosed-by", OPT_O_ENC,
+  {"fields-optionally-enclosed-by", 0,
    "Fields in the input file are optionally enclosed by the given character.", 
    &opt_enclosed, &opt_enclosed, 0, 
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"fields-escaped-by", OPT_ESC, 
+  {"fields-escaped-by", 0, 
    "Fields in the input file are escaped by the given character.",
    &escaped, &escaped, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0,
    0, 0},
@@ -126,10 +126,10 @@ static struct my_option my_long_options[] =
     "Disable foreign key checks while importing the data.",
     &ignore_foreign_keys, &ignore_foreign_keys, 0, GET_BOOL, NO_ARG,
     0, 0, 0, 0, 0, 0},
-  {"ignore-lines", OPT_IGN_LINES, "Ignore first n lines of data infile.",
+  {"ignore-lines", 0, "Ignore first n lines of data infile.",
    &opt_ignore_lines, &opt_ignore_lines, 0, GET_LL,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"lines-terminated-by", OPT_LTB, 
+  {"lines-terminated-by", 0, 
    "Lines in the input file are terminated by the given string.",
    &lines_terminated, &lines_terminated, 0, GET_STR,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -138,7 +138,7 @@ static struct my_option my_long_options[] =
   {"lock-tables", 'l', "Lock all tables for write (this disables threads).",
     &lock_tables, &lock_tables, 0, GET_BOOL, NO_ARG, 
     0, 0, 0, 0, 0, 0},
-  {"low-priority", OPT_LOW_PRIORITY,
+  {"low-priority", 0,
    "Use LOW_PRIORITY when updating the table.", &opt_low_priority,
    &opt_low_priority, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"password", 'p',
@@ -151,7 +151,7 @@ static struct my_option my_long_options[] =
   {"parallel", 'j', "Number of LOAD DATA jobs executed in parallel",
    &opt_use_threads, &opt_use_threads, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0,
    0, 0},
-  {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
+  {"plugin_dir", 0, "Directory for client-side plugins.",
    &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"port", 'P', "Port number to use for connection or 0 for default to, in "
@@ -173,7 +173,7 @@ static struct my_option my_long_options[] =
    &opt_mysql_unix_port, &opt_mysql_unix_port, 0, GET_STR,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 #include <sslopt-longopts.h>
-  {"use-threads", OPT_USE_THREADS, "Synonym for --parallel option",
+  {"use-threads", 0, "Synonym for --parallel option",
    &opt_use_threads, &opt_use_threads, 0,
    GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 #ifndef DONT_ALLOW_USER_CHANGE

client/mysqlshow.c

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2015, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2019, MariaDB
+   Copyright (c) 2010, 2024, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -180,10 +180,10 @@ static struct my_option my_long_options[] =
   {"character-sets-dir", 'c', "Directory for character set files.",
    (char**) &charsets_dir, (char**) &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0,
    0, 0, 0, 0, 0},
-  {"default-character-set", OPT_DEFAULT_CHARSET,
+  {"default-character-set", 0,
    "Set the default character set.", &default_charset,
    &default_charset, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"count", OPT_COUNT,
+  {"count", 0,
    "Show number of rows per table (may be slow for non-MyISAM tables).",
    &opt_count, &opt_count, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
@@ -192,13 +192,13 @@ static struct my_option my_long_options[] =
    0, 0, 0},
   {"debug", '#', "Output debug log. Often this is 'd:t:o,filename'.",
    0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit.",
+  {"debug-check", 0, "Check memory and open file usage at exit.",
    &debug_check_flag, &debug_check_flag, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"debug-info", OPT_DEBUG_INFO, "Print some debug info at exit.",
+  {"debug-info", 0, "Print some debug info at exit.",
    &debug_info_flag, &debug_info_flag,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"default_auth", OPT_DEFAULT_AUTH,
+  {"default_auth", 0,
    "Default authentication client-side plugin to use.",
    &opt_default_auth, &opt_default_auth, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -215,7 +215,7 @@ static struct my_option my_long_options[] =
    "Password to use when connecting to server. If password is not given, it's "
    "solicited on the tty.",
    0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
-  {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
+  {"plugin_dir", 0, "Directory for client-side plugins.",
    &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"port", 'P', "Port number to use for connection or 0 for default to, in "

client/mysqlslap.c

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2005, 2015, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2022, MariaDB
+   Copyright (c) 2010, 2024, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -524,50 +524,45 @@ static struct my_option my_long_options[] =
     "Generate SQL where not supplied by file or command line.",
     &auto_generate_sql, &auto_generate_sql,
     0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"auto-generate-sql-add-autoincrement", OPT_SLAP_AUTO_GENERATE_ADD_AUTO,
+  {"auto-generate-sql-add-autoincrement", 0,
     "Add an AUTO_INCREMENT column to auto-generated tables.",
-    &auto_generate_sql_autoincrement,
+    &auto_generate_sql_autoincrement, &auto_generate_sql_autoincrement,
-    &auto_generate_sql_autoincrement,
     0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"auto-generate-sql-execute-number", OPT_SLAP_AUTO_GENERATE_EXECUTE_QUERIES,
+  {"auto-generate-sql-execute-number", 0,
     "Set this number to generate a set number of queries to run.",
     &auto_actual_queries, &auto_actual_queries,
     0, GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"auto-generate-sql-guid-primary", OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY,
+  {"auto-generate-sql-guid-primary", 0,
     "Add GUID based primary keys to auto-generated tables.",
-    &auto_generate_sql_guid_primary,
+    &auto_generate_sql_guid_primary, &auto_generate_sql_guid_primary,
-    &auto_generate_sql_guid_primary,
     0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"auto-generate-sql-load-type", OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE,
+  {"auto-generate-sql-load-type", 0,
     "Specify test load type: mixed, update, write, key, or read; default is mixed.",
    (char**) &auto_generate_sql_type, (char**) &auto_generate_sql_type,
     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"auto-generate-sql-secondary-indexes", 
+  {"auto-generate-sql-secondary-indexes", 0,
-    OPT_SLAP_AUTO_GENERATE_SECONDARY_INDEXES, 
     "Number of secondary indexes to add to auto-generated tables.",
     &auto_generate_sql_secondary_indexes,
     &auto_generate_sql_secondary_indexes, 0,
     GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"auto-generate-sql-unique-query-number", 
+  {"auto-generate-sql-unique-query-number", 0,
-    OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM,
     "Number of unique queries to generate for automatic tests.",
     &auto_generate_sql_unique_query_number,
     &auto_generate_sql_unique_query_number,
     0, GET_ULL, REQUIRED_ARG, 10, 0, 0, 0, 0, 0},
-  {"auto-generate-sql-unique-write-number", 
+  {"auto-generate-sql-unique-write-number", 0,
-    OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM,
     "Number of unique queries to generate for auto-generate-sql-write-number.",
     &auto_generate_sql_unique_write_number,
     &auto_generate_sql_unique_write_number,
     0, GET_ULL, REQUIRED_ARG, 10, 0, 0, 0, 0, 0},
-  {"auto-generate-sql-write-number", OPT_SLAP_AUTO_GENERATE_WRITE_NUM,
+  {"auto-generate-sql-write-number", 0,
     "Number of row inserts to perform for each thread (default is 100).",
     &auto_generate_sql_number, &auto_generate_sql_number,
     0, GET_ULL, REQUIRED_ARG, 100, 0, 0, 0, 0, 0},
   {"character-sets-dir", OPT_CHARSETS_DIR,
    "Directory for character set files.", (char **)&charsets_dir,
    (char **)&charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"commit", OPT_SLAP_COMMIT, "Commit records every X number of statements.",
+  {"commit", 0, "Commit records every X number of statements.",
     &commit_rate, &commit_rate, 0, GET_UINT, REQUIRED_ARG,
     0, 0, 0, 0, 0, 0},
   {"compress", 'C', "Use compression in server/client protocol.",
@@ -576,10 +571,10 @@ static struct my_option my_long_options[] =
   {"concurrency", 'c', "Number of clients to simulate for query to run.",
    (char**) &concurrency_str, (char**) &concurrency_str, 0, GET_STR,
     REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"create", OPT_SLAP_CREATE_STRING, "File or string to use create tables.",
+  {"create", 0, "File or string to use create tables.",
     &create_string, &create_string, 0, GET_STR, REQUIRED_ARG,
     0, 0, 0, 0, 0, 0},
-  {"create-schema", OPT_CREATE_SLAP_SCHEMA, "Schema to run tests in.",
+  {"create-schema", 0, "Schema to run tests in.",
    (char**) &create_schema_string, (char**) &create_schema_string, 0, GET_STR, 
     REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"csv", OPT_SLAP_CSV,
@@ -593,12 +588,12 @@ static struct my_option my_long_options[] =
    (char**) &default_dbug_option, (char**) &default_dbug_option, 0, GET_STR,
     OPT_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"debug-check", OPT_DEBUG_CHECK, "Check memory and open file usage at exit.",
+  {"debug-check", 0, "Check memory and open file usage at exit.",
    &debug_check_flag, &debug_check_flag, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"debug-info", 'T', "Print some debug info at exit.", &debug_info_flag,
    &debug_info_flag, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"default_auth", OPT_DEFAULT_AUTH,
+  {"default_auth", 0,
    "Default authentication client-side plugin to use.",
    &opt_default_auth, &opt_default_auth, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -606,7 +601,7 @@ static struct my_option my_long_options[] =
     "Delimiter to use in SQL statements supplied in file or command line.",
    (char**) &delimiter, (char**) &delimiter, 0, GET_STR, REQUIRED_ARG,
     0, 0, 0, 0, 0, 0},
-  {"detach", OPT_SLAP_DETACH,
+  {"detach", 0,
     "Detach (close and reopen) connections after X number of requests.",
     &detach_rate, &detach_rate, 0, GET_UINT, REQUIRED_ARG, 
     0, 0, 0, 0, 0, 0},
@@ -618,14 +613,14 @@ static struct my_option my_long_options[] =
     GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"host", 'h', "Connect to host.", &host, &host, 0, GET_STR,
     REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"init-command", OPT_INIT_COMMAND,
+  {"init-command", 0,
    "SQL Command to execute when connecting to MariaDB server. Will "
    "automatically be re-executed when reconnecting.",
    &opt_init_command, &opt_init_command, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"iterations", 'i', "Number of times to run the tests.", &iterations,
     &iterations, 0, GET_UINT, REQUIRED_ARG, 1, 0, 0, 0, 0, 0},
-  {"no-drop", OPT_SLAP_NO_DROP, "Do not drop the schema after the test.",
+  {"no-drop", 0, "Do not drop the schema after the test.",
    &opt_no_drop, &opt_no_drop, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"number-char-cols", 'x', 
     "Number of VARCHAR columns to create in table if specifying --auto-generate-sql.",
@@ -635,11 +630,11 @@ static struct my_option my_long_options[] =
     "Number of INT columns to create in table if specifying --auto-generate-sql.",
    (char**) &num_int_cols_opt, (char**) &num_int_cols_opt, 0, GET_STR, REQUIRED_ARG, 
     0, 0, 0, 0, 0, 0},
-  {"number-of-queries", OPT_MYSQL_NUMBER_OF_QUERY, 
+  {"number-of-queries", 0,
     "Limit each client to this number of queries (this is not exact).",
     &num_of_query, &num_of_query, 0,
     GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"only-print", OPT_MYSQL_ONLY_PRINT,
+  {"only-print", 0,
     "Do not connect to the databases, but instead print out what would have "
      "been done.",
     &opt_only_print, &opt_only_print, 0, GET_BOOL,  NO_ARG,
@@ -651,25 +646,25 @@ static struct my_option my_long_options[] =
   {"pipe", 'W', "Use named pipes to connect to server.", 0, 0, 0, GET_NO_ARG,
     NO_ARG, 0, 0, 0, 0, 0, 0},
 #endif
-  {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
+  {"plugin_dir", 0, "Directory for client-side plugins.",
    &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"port", 'P', "Port number to use for connection.", &opt_mysql_port,
     &opt_mysql_port, 0, GET_UINT, REQUIRED_ARG, MYSQL_PORT, 0, 0, 0, 0,
     0},
-  {"post-query", OPT_SLAP_POST_QUERY,
+  {"post-query", 0,
     "Query to run or file containing query to execute after tests have completed.",
     &user_supplied_post_statements, &user_supplied_post_statements,
     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"post-system", OPT_SLAP_POST_SYSTEM,
+  {"post-system", 0,
     "system() string to execute after tests have completed.",
     &post_system, &post_system,
     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"pre-query", OPT_SLAP_PRE_QUERY, 
+  {"pre-query", 0,
     "Query to run or file containing query to execute before running tests.",
     &user_supplied_pre_statements, &user_supplied_pre_statements,
     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"pre-system", OPT_SLAP_PRE_SYSTEM, 
+  {"pre-system", 0,
     "system() string to execute before running tests.",
     &pre_system, &pre_system,
     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -1631,6 +1626,9 @@ drop_primary_key_list(void)
   return 0;
 }
 
+
+PRAGMA_DISABLE_CHECK_STACK_FRAME
+
 static int
 create_schema(MYSQL *mysql, const char *db, statement *stmt, 
               option_string *engine_stmt)
@@ -1726,6 +1724,7 @@ limit_not_met:
 
   DBUG_RETURN(0);
 }
+PRAGMA_REENABLE_CHECK_STACK_FRAME
 
 static int
 drop_schema(MYSQL *mysql, const char *db)

client/mysqltest.cc

@@ -78,7 +78,7 @@ static my_bool non_blocking_api_enabled= 0;
 #define MAX_DELIMITER_LENGTH 16
 #define DEFAULT_MAX_CONN        64
 
-#define DIE_BUFF_SIZE           256*1024
+#define DIE_BUFF_SIZE           15*1024
 
 #define RESULT_STRING_INIT_MEM 2048
 #define RESULT_STRING_INCREMENT_MEM 2048
@@ -404,7 +404,7 @@ enum enum_commands {
   Q_IF,
   Q_DISABLE_PARSING, Q_ENABLE_PARSING,
   Q_REPLACE_REGEX, Q_REMOVE_FILE, Q_FILE_EXIST,
-  Q_WRITE_FILE, Q_COPY_FILE, Q_PERL, Q_DIE, Q_EXIT, Q_SKIP,
+  Q_WRITE_FILE, Q_WRITE_LINE, Q_COPY_FILE, Q_PERL, Q_DIE, Q_EXIT, Q_SKIP,
   Q_CHMOD_FILE, Q_APPEND_FILE, Q_CAT_FILE, Q_DIFF_FILES,
   Q_SEND_QUIT, Q_CHANGE_USER, Q_MKDIR, Q_RMDIR,
   Q_LIST_FILES, Q_LIST_FILES_WRITE_FILE, Q_LIST_FILES_APPEND_FILE,
@@ -508,6 +508,7 @@ const char *command_names[]=
   "remove_file",
   "file_exists",
   "write_file",
+  "write_line",
   "copy_file",
   "perl",
   "die",
@@ -623,7 +624,7 @@ void replace_strings_append(struct st_replace *rep, DYNAMIC_STRING* ds,
 const char *from);
 
 ATTRIBUTE_NORETURN
-static void cleanup_and_exit(int exit_code);
+static void cleanup_and_exit(int exit_code, bool called_from_die);
 
 ATTRIBUTE_NORETURN
 static void really_die(const char *msg);
@@ -941,6 +942,7 @@ pthread_attr_t cn_thd_attrib;
 pthread_handler_t connection_thread(void *arg)
 {
   struct st_connection *cn= (struct st_connection*)arg;
+  DBUG_ENTER("connection_thread");
 
   mysql_thread_init();
   while (cn->command != EMB_END_CONNECTION)
@@ -952,6 +954,7 @@ pthread_handler_t connection_thread(void *arg)
         pthread_cond_wait(&cn->query_cond, &cn->query_mutex);
       pthread_mutex_unlock(&cn->query_mutex);
     }
+    DBUG_PRINT("info", ("executing command: %d", cn->command));
     switch (cn->command)
     {
       case EMB_END_CONNECTION:
@@ -972,24 +975,26 @@ pthread_handler_t connection_thread(void *arg)
         break;
       case EMB_CLOSE_STMT:
         cn->result= mysql_stmt_close(cn->stmt);
+        cn->stmt= 0;
         break;
       default:
         DBUG_ASSERT(0);
     }
-    cn->command= 0;
     pthread_mutex_lock(&cn->result_mutex);
     cn->query_done= 1;
+    cn->command= 0;
     pthread_cond_signal(&cn->result_cond);
     pthread_mutex_unlock(&cn->result_mutex);
   }
 
 end_thread:
-  cn->query_done= 1;
+  DBUG_ASSERT(cn->stmt == 0);
   mysql_close(cn->mysql);
   cn->mysql= 0;
+  cn->query_done= 1;
   mysql_thread_end();
   pthread_exit(0);
-  return 0;
+  DBUG_RETURN(0);
 }
 
 static void wait_query_thread_done(struct st_connection *con)
@@ -1007,12 +1012,16 @@ static void wait_query_thread_done(struct st_connection *con)
 
 static void signal_connection_thd(struct st_connection *cn, int command)
 {
+  DBUG_ENTER("signal_connection_thd");
+  DBUG_PRINT("enter", ("command: %d", command));
+
   DBUG_ASSERT(cn->has_thread);
   cn->query_done= 0;
-  cn->command= command;
   pthread_mutex_lock(&cn->query_mutex);
+  cn->command= command;
   pthread_cond_signal(&cn->query_cond);
   pthread_mutex_unlock(&cn->query_mutex);
+  DBUG_VOID_RETURN;
 }
 
 
@@ -1077,27 +1086,37 @@ static int do_stmt_execute(struct st_connection *cn)
 static int do_stmt_close(struct st_connection *cn)
 {
   DBUG_ENTER("do_stmt_close");
-  /* The cn->stmt is already set. */
   if (!cn->has_thread)
-    DBUG_RETURN(mysql_stmt_close(cn->stmt));
+  {
+    /* The cn->stmt is already set. */
+    int res= mysql_stmt_close(cn->stmt);
+    cn->stmt= 0;
+    DBUG_RETURN(res);
+  }
+  wait_query_thread_done(cn);
   signal_connection_thd(cn, EMB_CLOSE_STMT);
   wait_query_thread_done(cn);
+  DBUG_ASSERT(cn->stmt == 0);
   DBUG_RETURN(cn->result);
 }
 
 
 static void emb_close_connection(struct st_connection *cn)
 {
+  DBUG_ENTER("emb_close_connection");
   if (!cn->has_thread)
-    return;
+    DBUG_VOID_RETURN;
   wait_query_thread_done(cn);
   signal_connection_thd(cn, EMB_END_CONNECTION);
   pthread_join(cn->tid, NULL);
   cn->has_thread= FALSE;
+  DBUG_ASSERT(cn->mysql == 0);
+  DBUG_ASSERT(cn->stmt == 0);
   pthread_mutex_destroy(&cn->query_mutex);
   pthread_cond_destroy(&cn->query_cond);
   pthread_mutex_destroy(&cn->result_mutex);
   pthread_cond_destroy(&cn->result_cond);
+  DBUG_VOID_RETURN;
 }
 
 
@@ -1121,7 +1140,13 @@ static void init_connection_thd(struct st_connection *cn)
 #define do_read_query_result(cn) mysql_read_query_result(cn->mysql)
 #define do_stmt_prepare(cn, q, q_len) mysql_stmt_prepare(cn->stmt, q, (ulong)q_len)
 #define do_stmt_execute(cn) mysql_stmt_execute(cn->stmt)
-#define do_stmt_close(cn) mysql_stmt_close(cn->stmt)
+
+static int do_stmt_close(struct st_connection *cn)
+{
+  int res= mysql_stmt_close(cn->stmt);
+  cn->stmt= 0;
+  return res;
+}
 
 #endif /*EMBEDDED_LIBRARY*/
 
@@ -1449,7 +1474,6 @@ void close_statements()
   {
     if (con->stmt)
       do_stmt_close(con);
-    con->stmt= 0;
   }
   DBUG_VOID_RETURN;
 }
@@ -1520,8 +1544,8 @@ void free_used_memory()
 void ha_pre_shutdown();
 #endif
 
-
+ATTRIBUTE_NORETURN static void cleanup_and_exit(int exit_code,
-ATTRIBUTE_NORETURN static void cleanup_and_exit(int exit_code)
+                                                bool called_from_die)
 {
 #ifdef EMBEDDED_LIBRARY
   if (server_initialized)
@@ -1534,16 +1558,6 @@ ATTRIBUTE_NORETURN static void cleanup_and_exit(int exit_code)
   if (server_initialized)
     mysql_server_end();
 
-  /*
-    mysqltest is fundamentally written in a way that makes impossible
-    to free all memory before exit (consider memory allocated
-    for frame local DYNAMIC_STRING's and die() invoked down the stack.
-
-    We close stderr here to stop unavoidable safemalloc reports
-    from polluting the output.
-  */
-  fclose(stderr);
-
   my_end(my_end_arg);
 
   if (!silent) {
@@ -1563,6 +1577,11 @@ ATTRIBUTE_NORETURN static void cleanup_and_exit(int exit_code)
     }
   }
 
+  /*
+    Report memory leaks, if not called from 'die()', as die() will not release
+    all memory.
+  */
+  sf_leaking_memory= called_from_die;
   exit(exit_code);
 }
 
@@ -1629,7 +1648,7 @@ static void really_die(const char *msg)
     second time, just exit
   */
   if (dying)
-    cleanup_and_exit(1);
+    cleanup_and_exit(1, 1);
   dying= 1;
 
   log_file.show_tail(opt_tail_lines);
@@ -1641,7 +1660,7 @@ static void really_die(const char *msg)
   if (cur_con && !cur_con->pending)
     show_warnings_before_error(cur_con->mysql);
 
-  cleanup_and_exit(1);
+  cleanup_and_exit(1, 1);
 }
 
 void report_or_die(const char *fmt, ...)
@@ -1695,7 +1714,7 @@ void abort_not_supported_test(const char *fmt, ...)
   }
   va_end(args);
 
-  cleanup_and_exit(62);
+  cleanup_and_exit(62, 0);
 }
 
 
@@ -2242,14 +2261,14 @@ int dyn_string_cmp(DYNAMIC_STRING* ds, const char *fname)
   check_result
 
   RETURN VALUES
-  error - the function will not return
+  0  ok
-
+  1  error
 */
 
-void check_result()
+int check_result()
 {
   const char *mess= 0;
-
+  int error= 1;
   DBUG_ENTER("check_result");
   DBUG_ASSERT(result_file_name);
   DBUG_PRINT("enter", ("result_file_name: %s", result_file_name));
@@ -2257,7 +2276,10 @@ void check_result()
   switch (compare_files(log_file.file_name(), result_file_name)) {
   case RESULT_OK:
     if (!error_count)
+    {
+      error= 0;
       break; /* ok */
+    }
     mess= "Got errors while running test";
     /* Fallthrough */
   case RESULT_LENGTH_MISMATCH:
@@ -2296,14 +2318,13 @@ void check_result()
           log_file.file_name(), reject_file, errno);
 
     show_diff(NULL, result_file_name, reject_file);
-    die("%s", mess);
+    fprintf(stderr, "%s", mess);
     break;
   }
   default: /* impossible */
     die("Unknown error code from dyn_string_cmp()");
   }
-
+  DBUG_RETURN(error);
-  DBUG_VOID_RETURN;
 }
 
 
@@ -4357,6 +4378,49 @@ void do_write_file(struct st_command *command)
   do_write_file_command(command, FALSE);
 }
 
+/**
+  Write a line to the start of the file.
+  Truncates existing file, creates new one if it doesn't exist.
+
+  Usage
+  write_line <line> <filename>;
+
+  Example
+  --write_line restart $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+
+  @note Both the file and the line parameters are evaluated
+  (can be variables).
+
+  @note This is a better alternative to
+  exec echo > file, as it doesn't depend on shell,
+  and can better handle sporadic file access errors caused
+  by antivirus or backup software on Windows.
+*/
+void do_write_line(struct st_command *command)
+{
+  DYNAMIC_STRING ds_line;
+  DYNAMIC_STRING ds_filename;
+
+  struct command_arg write_line_args[] = {
+    { "line", ARG_STRING, FALSE, &ds_line, "line to add" },
+    { "filename", ARG_STRING, TRUE, &ds_filename, "File to write to" },
+  };
+  DBUG_ENTER("do_write_line");
+
+  check_command_args(command,
+                     command->first_argument,
+                     write_line_args,
+                     sizeof(write_line_args)/sizeof(struct command_arg),
+                     ' ');
+
+  if (bad_path(ds_filename.str))
+    DBUG_VOID_RETURN;
+  dynstr_append_mem(&ds_line, "\n", 1);
+  str_to_file2(ds_filename.str, ds_line.str, ds_line.length, FALSE);
+  dynstr_free(&ds_filename);
+  dynstr_free(&ds_line);
+  DBUG_VOID_RETURN;
+}
 
 /*
   SYNOPSIS
@@ -5303,7 +5367,11 @@ void do_shutdown_server(struct st_command *command)
   */
 
   if (timeout && mysql_shutdown(mysql, SHUTDOWN_DEFAULT))
-    die("mysql_shutdown failed");
+  {
+    handle_error(command, mysql_errno(mysql), mysql_error(mysql),
+                 mysql_sqlstate(mysql), &ds_res);
+    DBUG_VOID_RETURN;
+  }
 
   if (!timeout || wait_until_dead(pid, timeout))
   {
@@ -5731,7 +5799,6 @@ void do_close_connection(struct st_command *command)
 #endif /*!EMBEDDED_LIBRARY*/
   if (con->stmt)
     do_stmt_close(con);
-  con->stmt= 0;
 #ifdef EMBEDDED_LIBRARY
   /*
     As query could be still executed in a separate thread
@@ -7435,17 +7502,17 @@ get_one_option(const struct my_option *opt, const char *argument, const char *)
     break;
   case 'V':
     print_version();
-    exit(0);
+    cleanup_and_exit(0,0);
   case OPT_MYSQL_PROTOCOL:
 #ifndef EMBEDDED_LIBRARY
     if ((opt_protocol= find_type_with_warning(argument, &sql_protocol_typelib,
                                               opt->name)) <= 0)
-      exit(1);
+      cleanup_and_exit(1,0);
 #endif
     break;
   case '?':
     usage();
-    exit(0);
+    cleanup_and_exit(0,0);
   }
   return 0;
 }
@@ -7457,12 +7524,12 @@ int parse_args(int argc, char **argv)
   default_argv= argv;
 
   if ((handle_options(&argc, &argv, my_long_options, get_one_option)))
-    exit(1);
+    cleanup_and_exit(1, 0);
 
   if (argc > 1)
   {
     usage();
-    exit(1);
+    cleanup_and_exit(1, 0);
   }
   if (argc == 1)
     opt_db= *argv;
@@ -7529,7 +7596,7 @@ void str_to_file2(const char *fname, char *str, size_t size, my_bool append)
     die("Could not open '%s' for writing, errno: %d", buff, errno);
   if (append && my_seek(fd, 0, SEEK_END, MYF(0)) == MY_FILEPOS_ERROR)
     die("Could not find end of file '%s', errno: %d", buff, errno);
-  if (my_write(fd, (uchar*)str, size, MYF(MY_WME|MY_FNABP)))
+  if (size > 0 && my_write(fd, (uchar*)str, size, MYF(MY_WME|MY_FNABP)))
     die("write failed, errno: %d", errno);
   my_close(fd, MYF(0));
 }
@@ -8326,7 +8393,7 @@ static int match_expected_error(struct st_command *command,
 
   SYNOPSIS
   handle_error()
-  q     - query context
+  command   - command
   err_errno - error number
   err_error - error message
   err_sqlstate - sql state
@@ -8585,7 +8652,7 @@ void run_query_stmt(struct st_connection *cn, struct st_command *command,
   my_bool ds_res_1st_execution_init = FALSE;
   my_bool compare_2nd_execution = TRUE;
   int query_match_ps2_re;
-
+  MYSQL_RES *res;
   DBUG_ENTER("run_query_stmt");
   DBUG_PRINT("query", ("'%-.60s'", query));
   DBUG_PRINT("info",
@@ -8791,10 +8858,13 @@ void run_query_stmt(struct st_connection *cn, struct st_command *command,
         The --enable_prepare_warnings command can be used to change this so
         that warnings from both the prepare and execute phase are shown.
       */
-      if ((mysql_stmt_result_metadata(stmt) != NULL) &&
+      if ((res= mysql_stmt_result_metadata(stmt)))
-          !disable_warnings &&
+      {
+        if (!disable_warnings &&
             !prepare_warnings_enabled)
           dynstr_set(&ds_prepare_warnings, NULL);
+        mysql_free_result(res);
+      }
 
       /*
         Fetch info before fetching warnings, since it will be reset
@@ -9923,6 +9993,7 @@ static sig_handler signal_handler(int sig)
   fflush(stderr);
   my_write_core(sig);
 #ifndef _WIN32
+  sf_leaking_memory= 1;
   exit(1);			// Shouldn't get here but just in case
 #endif
 }
@@ -9996,12 +10067,10 @@ int main(int argc, char **argv)
   uint command_executed= 0, last_command_executed= 0;
   char save_file[FN_REFLEN];
   bool empty_result= FALSE;
+  int error= 0;
   MY_INIT(argv[0]);
   DBUG_ENTER("main");
 
-  /* mysqltest has no way to free all its memory correctly */
-  sf_leaking_memory= 1;
-
   save_file[0]= 0;
   TMPDIR[0]= 0;
 
@@ -10316,6 +10385,7 @@ int main(int argc, char **argv)
         break;
       case Q_FILE_EXIST: do_file_exist(command); break;
       case Q_WRITE_FILE: do_write_file(command); break;
+      case Q_WRITE_LINE: do_write_line(command); break;
       case Q_APPEND_FILE: do_append_file(command); break;
       case Q_DIFF_FILES: do_diff_files(command); break;
       case Q_SEND_QUIT: do_send_quit(command); break;
@@ -10683,7 +10753,7 @@ int main(int argc, char **argv)
     die("Test ended with parsing disabled");
 
   /*
-    The whole test has been executed _successfully_.
+    The whole test has been executed successfully.
     Time to compare result or save it to record file.
     The entire output from test is in the log file
   */
@@ -10706,7 +10776,7 @@ int main(int argc, char **argv)
       else
       {
 	/* Check that the output from test is equal to result file */
-	check_result();
+	error= check_result();
       }
     }
   }
@@ -10716,7 +10786,8 @@ int main(int argc, char **argv)
     if (! result_file_name || record ||
         compare_files (log_file.file_name(), result_file_name))
     {
-      die("The test didn't produce any output");
+      fprintf(stderr, "mysqltest: The test didn't produce any output\n");
+      error= 1;
     }
     else 
     {
@@ -10725,12 +10796,15 @@ int main(int argc, char **argv)
   }
 
   if (!command_executed && result_file_name && !empty_result)
-    die("No queries executed but non-empty result file found!");
+  {
+    fprintf(stderr, "mysqltest: No queries executed but non-empty result file found!\n");
+    error= 1;
+  }
 
+  if (!error)
     verbose_msg("Test has succeeded!");
   timer_output();
-  /* Yes, if we got this far the test has succeeded! Sakila smiles */
+  cleanup_and_exit(error, 0);
-  cleanup_and_exit(0);
   return 0; /* Keep compiler happy too */
 }
 

cmake/FindPMEM.cmake

@@ -1,18 +0,0 @@
-if(PMEM_LIBRARIES)
-  set(PMEM_FOUND TRUE)
-  return()
-endif()
-if(DEFINED PMEM_LIBRARIES)
-  set(PMEM_FOUND FALSE)
-  return()
-endif()
-
-find_path(PMEM_INCLUDE_DIRS NAMES libpmem.h)
-find_library(PMEM_LIBRARIES NAMES pmem)
-
-include(FindPackageHandleStandardArgs)
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(
-    PMEM DEFAULT_MSG
-    PMEM_LIBRARIES PMEM_INCLUDE_DIRS)
-
-mark_as_advanced(PMEM_INCLUDE_DIRS PMEM_LIBRARIES)

cmake/libutils.cmake

@@ -379,5 +379,11 @@ FUNCTION (MAYBE_DISABLE_IPO target)
       INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF
       INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO OFF
       INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL OFF)
+    IF(CMAKE_CONFIGURATION_TYPES)
+      FOREACH(cfg ${CMAKE_CONFIGURATION_TYPES})
+        STRING(TOUPPER "${cfg}" cfg_upper)
+        SET_TARGET_PROPERTIES(${target} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_${cfg_upper} OFF)
+      ENDFOREACH()
+    ENDIF()
   ENDIF()
 ENDFUNCTION()

cmake/mariadb_connector_c.cmake

@@ -40,6 +40,13 @@ SET(CLIENT_PLUGIN_PVIO_SOCKET STATIC)
 MESSAGE("== Configuring MariaDB Connector/C")
 ADD_SUBDIRECTORY(libmariadb)
 
+IF(MSVC AND TARGET mariadb_obj AND TARGET mariadbclient)
+  # With MSVC, do not produce LTCG-compiled static client libraries.
+  # They are not usable by end-users, being tied to exact compiler version
+  MAYBE_DISABLE_IPO(mariadb_obj)
+  MAYBE_DISABLE_IPO(mariadbclient)
+ENDIF()
+
 IF(UNIX)
   INSTALL(CODE "EXECUTE_PROCESS(
                   COMMAND ${CMAKE_COMMAND} -E make_directory \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR})

cmake/os/AIX.cmake

@@ -34,8 +34,5 @@ ELSE()
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -maix64 -pthread -mcmodel=large")
 ENDIF()
 
-# fcntl(fd, F_SETFL, O_DIRECT) is not supported; O_DIRECT is an open(2) flag
-SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "")
-
 # make it WARN by default, not AUTO (that implies -Werror)
 SET(MYSQL_MAINTAINER_MODE "WARN" CACHE STRING "Enable MariaDB maintainer-specific warnings. One of: NO (warnings are disabled) WARN (warnings are enabled) ERR (warnings are errors) AUTO (warnings are errors in Debug only)")

cmake/os/SunOS.cmake

@@ -17,10 +17,6 @@ INCLUDE(CheckSymbolExists)
 INCLUDE(CheckCSourceRuns)
 INCLUDE(CheckCSourceCompiles) 
 
-# fcntl(fd, F_SETFL, O_DIRECT) is not supported,
-# and directio(3C) would only work on UFS or NFS, not ZFS.
-SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "")
-
 # Enable 64 bit file offsets
 SET(_FILE_OFFSET_BITS 64)
 

cmake/os/WindowsCache.cmake

@@ -43,7 +43,6 @@ SET(HAVE_EXECINFO_H CACHE  INTERNAL "")
 SET(HAVE_FCHMOD CACHE  INTERNAL "")
 SET(HAVE_FCNTL CACHE  INTERNAL "")
 SET(HAVE_FCNTL_H 1 CACHE  INTERNAL "")
-SET(HAVE_FCNTL_DIRECT 0 CACHE  INTERNAL "")
 SET(HAVE_FCNTL_NONBLOCK CACHE  INTERNAL "")
 SET(HAVE_FDATASYNC CACHE  INTERNAL "")
 SET(HAVE_DECL_FDATASYNC CACHE INTERNAL "")

cmake/pcre.cmake

@@ -4,6 +4,9 @@ SET(WITH_PCRE "auto" CACHE STRING
    "Which pcre to use (possible values are 'bundled', 'system', or 'auto')")
 
 MACRO(BUNDLE_PCRE2)
+  SET(WITH_PCRE "bundled" CACHE STRING
+   "Which pcre to use (possible values are 'bundled', 'system', or 'auto')")
+
   SET(dir "${CMAKE_BINARY_DIR}/extra/pcre2")
   SET(PCRE_INCLUDE_DIRS ${dir}/src/pcre2-build ${dir}/src/pcre2/src)
   MESSAGE(STATUS "Will download and bundle pcre2")
@@ -41,21 +44,21 @@ MACRO(BUNDLE_PCRE2)
     SET(byproducts ${byproducts} BUILD_BYPRODUCTS ${file} ${file_d})
     SET_TARGET_PROPERTIES(${lib} PROPERTIES IMPORTED_LOCATION ${file})
   ENDFOREACH()
+
   FOREACH(v "" "_DEBUG" "_RELWITHDEBINFO" "_RELEASE" "_MINSIZEREL")
-    STRING(REPLACE "/WX" "" pcre2_flags${v} "${CMAKE_C_FLAGS${v}}")
+    SET(pcre2_flags${v} "${CMAKE_C_FLAGS${v}}")
-    SET(pcre2_flags${v} "${pcre2_flags${v}} -std=c99 ")
     IF(MSVC)
+      STRING(REPLACE "/WX" "" pcre2_flags${v} "${pcre2_flags${v}}")
       # Suppress a warning
-      STRING(APPEND pcre2_flags${v} " /wd4244 " )
+      STRING(APPEND pcre2_flags${v} " /wd4244 /wd4267 " )
-      # Disable asan support
-      STRING(REPLACE "-fsanitize=address" "" pcre2_flags${v} "${CMAKE_C_FLAGS${v}}")
     ENDIF()
   ENDFOREACH()
+
   ExternalProject_Add(
     pcre2
     PREFIX   "${dir}"
-    URL "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.42/pcre2-10.42.zip"
+    URL "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.43/pcre2-10.43.zip"
-    URL_MD5 fe90992fbfb03f854bd9f344074f49eb
+    URL_MD5 b58f050f2fdd6f2ca5774a2975377a85
     INSTALL_COMMAND ""
     CMAKE_ARGS
       "-DCMAKE_WARN_DEPRECATED=FALSE"

cmake/plugin.cmake

@@ -46,7 +46,7 @@ MACRO(MYSQL_ADD_PLUGIN)
                     ${CMAKE_SOURCE_DIR}/sql
                     ${PCRE_INCLUDE_DIRS}
                     ${SSL_INCLUDE_DIRS}
-                    ${ZLIB_INCLUDE_DIR})
+                    ${ZLIB_INCLUDE_DIRS})
 
   LIST(GET ARG_UNPARSED_ARGUMENTS 0 plugin)
   SET(SOURCES ${ARG_UNPARSED_ARGUMENTS})

cmake/ssl.cmake

@@ -47,12 +47,13 @@ MACRO (MYSQL_USE_BUNDLED_SSL)
     ${CMAKE_SOURCE_DIR}/extra/wolfssl/wolfssl
     ${CMAKE_SOURCE_DIR}/extra/wolfssl/wolfssl/wolfssl
   )
-  SET(SSL_LIBRARIES  wolfssl wolfcrypt)
+  SET(SSL_LIBRARIES  wolfssl)
   SET(SSL_INCLUDE_DIRS ${INC_DIRS})
   SET(SSL_DEFINES "-DHAVE_OPENSSL -DHAVE_WOLFSSL  -DWOLFSSL_USER_SETTINGS")
   SET(HAVE_ERR_remove_thread_state ON CACHE INTERNAL "wolfssl doesn't have ERR_remove_thread_state")
   SET(HAVE_EncryptAes128Ctr ON CACHE INTERNAL "wolfssl does support AES-CTR")
   SET(HAVE_EncryptAes128Gcm OFF CACHE INTERNAL "wolfssl does not support AES-GCM")
+  SET(HAVE_des ON CACHE INTERNAL "wolfssl does support DES API")
   SET(HAVE_hkdf ON CACHE INTERNAL "wolfssl does support EVP_PKEY API")
   CHANGE_SSL_SETTINGS("bundled")
   ADD_SUBDIRECTORY(extra/wolfssl)
@@ -134,6 +135,8 @@ MACRO (MYSQL_CHECK_SSL)
                           HAVE_EncryptAes128Ctr)
       CHECK_SYMBOL_EXISTS(EVP_aes_128_gcm "openssl/evp.h"
                           HAVE_EncryptAes128Gcm)
+      CHECK_SYMBOL_EXISTS(DES_set_key_unchecked "openssl/des.h"
+                          HAVE_des)
       CHECK_SYMBOL_EXISTS(EVP_PKEY_CTX_set_hkdf_md "string.h;stdarg.h;openssl/kdf.h"
                           HAVE_hkdf)
       SET(CMAKE_REQUIRED_INCLUDES)

cmake/zlib.cmake

@@ -14,9 +14,12 @@
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1335  USA 
 
 MACRO (MYSQL_USE_BUNDLED_ZLIB)
-  SET(ZLIB_INCLUDE_DIR  ${CMAKE_SOURCE_DIR}/zlib ${CMAKE_BINARY_DIR}/zlib)
+  SET(ZLIB_INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/zlib ${CMAKE_BINARY_DIR}/zlib)
   SET(BUILD_BUNDLED_ZLIB 1)
-  SET(ZLIB_LIBRARY zlib CACHE INTERNAL "Bundled zlib library")
+  SET(ZLIB_LIBRARIES zlib CACHE INTERNAL "Bundled zlib library")
+  # temporarily define ZLIB_LIBRARY and ZLIB_INCLUDE_DIR for libmariadb
+  SET(ZLIB_LIBRARY ${ZLIB_LIBRARIES})
+  SET(ZLIB_INCLUDE_DIR ${ZLIB_INCLUDE_DIRS})
   SET(ZLIB_FOUND  TRUE)
   SET(WITH_ZLIB "bundled" CACHE STRING "Use bundled zlib")
   ADD_SUBDIRECTORY(zlib)
@@ -29,7 +32,7 @@ ENDMACRO()
 # If this is set,we use bundled zlib
 # If this is not set,search for system zlib. 
 # if system zlib is not found, use bundled copy
-# ZLIB_LIBRARIES, ZLIB_INCLUDE_DIR and ZLIB_SOURCES
+# ZLIB_LIBRARIES, ZLIB_INCLUDE_DIRS
 # are set after this macro has run
 
 MACRO (MYSQL_CHECK_ZLIB_WITH_COMPRESS)
@@ -37,10 +40,14 @@ MACRO (MYSQL_CHECK_ZLIB_WITH_COMPRESS)
   IF(WITH_ZLIB STREQUAL "bundled")
     MYSQL_USE_BUNDLED_ZLIB()
   ELSE()
-    INCLUDE(FindZLIB)
+    FIND_PACKAGE(PkgConfig QUIET)
+    IF(PKG_CONFIG_FOUND AND (COMMAND PKG_GET_VARIABLE) AND (NOT WIN32))
+      PKG_GET_VARIABLE(ZLIB_ROOT zlib prefix)
+    ENDIF()
+    FIND_PACKAGE(ZLIB)
     IF(ZLIB_FOUND)
      INCLUDE(CheckFunctionExists)
-      SET(CMAKE_REQUIRED_LIBRARIES z)
+      SET(CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES})
       CHECK_FUNCTION_EXISTS(crc32 HAVE_CRC32)
       CHECK_FUNCTION_EXISTS(compressBound HAVE_COMPRESSBOUND)
       CHECK_FUNCTION_EXISTS(deflateBound HAVE_DEFLATEBOUND)
@@ -48,7 +55,6 @@ MACRO (MYSQL_CHECK_ZLIB_WITH_COMPRESS)
       IF(HAVE_CRC32 AND HAVE_COMPRESSBOUND AND HAVE_DEFLATEBOUND)
         SET(WITH_ZLIB "system" CACHE STRING
           "Which zlib to use (possible values are 'bundled' or 'system')")
-        SET(ZLIB_SOURCES "")
       ELSE()
         SET(ZLIB_FOUND FALSE CACHE INTERNAL "Zlib found but not usable")
         MESSAGE(STATUS "system zlib found but not usable")

config.h.cmake

@@ -30,7 +30,6 @@
 #cmakedefine HAVE_DLFCN_H 1
 #cmakedefine HAVE_EXECINFO_H 1
 #cmakedefine HAVE_FCNTL_H 1
-#cmakedefine HAVE_FCNTL_DIRECT 1
 #cmakedefine HAVE_FENV_H 1
 #cmakedefine HAVE_FLOAT_H 1
 #cmakedefine HAVE_FNMATCH_H 1
@@ -499,6 +498,7 @@
 #cmakedefine HAVE_COMPRESS 1
 #cmakedefine HAVE_EncryptAes128Ctr 1
 #cmakedefine HAVE_EncryptAes128Gcm 1
+#cmakedefine HAVE_des 1
 #cmakedefine HAVE_hkdf 1
 
 /*

configure.cmake

@@ -60,15 +60,6 @@ IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" AND (NOT MSVC))
   ENDIF()
 ENDIF()
 
-# workaround for old gcc on x86, gcc atomic ops only work under -march=i686
-IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "i686" AND CMAKE_COMPILER_IS_GNUCC AND
-   CMAKE_C_COMPILER_VERSION VERSION_LESS "4.4.0")
-  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=i686")
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=i686")
-  # query_response_time.cc causes "error: unable to find a register to spill"
-  SET(PLUGIN_QUERY_RESPONSE_TIME NO CACHE BOOL "Disabled, gcc is too old")
-ENDIF()
-
 # use runtime atomic-support detection in aarch64
 IF(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
   MY_CHECK_AND_SET_COMPILER_FLAG("-moutline-atomics")
@@ -705,7 +696,6 @@ CHECK_SYMBOL_EXISTS(O_NONBLOCK "unistd.h;fcntl.h" HAVE_FCNTL_NONBLOCK)
 IF(NOT HAVE_FCNTL_NONBLOCK)
  SET(NO_FCNTL_NONBLOCK 1)
 ENDIF()
-CHECK_SYMBOL_EXISTS(O_DIRECT "fcntl.h" HAVE_FCNTL_DIRECT)
 
 #
 # Test for how the C compiler does inline, if at all
@@ -985,3 +975,8 @@ IF(have_C__Werror)
   )
   SET(CMAKE_REQUIRED_FLAGS ${SAVE_CMAKE_REQUIRED_FLAGS})
 ENDIF()
+
+IF(CMAKE_C_COMPILER_ID MATCHES "Intel")
+  MY_CHECK_AND_SET_COMPILER_FLAG("-no-ansi-alias")
+  MY_CHECK_AND_SET_COMPILER_FLAG("-fp-model precise")
+ENDIF()

debian/autobake-deb.sh

@@ -51,12 +51,6 @@ replace_uring_with_aio()
       -e '/-DWITH_URING=ON/d' -i debian/rules
 }
 
-disable_pmem()
-{
-  sed '/libpmem-dev/d' -i debian/control
-  sed '/-DWITH_PMEM=ON/d' -i debian/rules
-}
-
 disable_libfmt()
 {
   # 7.0+ required
@@ -96,10 +90,6 @@ in
   "buster")
     disable_libfmt
     replace_uring_with_aio
-    if [ ! "$architecture" = amd64 ]
-    then
-      disable_pmem
-    fi
     ;&
   "bullseye")
     add_lsb_base_depends
@@ -107,10 +97,6 @@ in
   "bookworm")
     # mariadb-plugin-rocksdb in control is 4 arches covered by the distro rocksdb-tools
     # so no removal is necessary.
-    if [[ ! "$architecture" =~ amd64|arm64|ppc64el ]]
-    then
-      disable_pmem
-    fi
     if [[ ! "$architecture" =~ amd64|arm64|armel|armhf|i386|mips64el|mipsel|ppc64el|s390x ]]
     then
       replace_uring_with_aio
@@ -129,10 +115,6 @@ in
     add_lsb_base_depends
     ;&
   "lunar"|"mantic")
-    if [[ ! "$architecture" =~ amd64|arm64|ppc64el ]]
-    then
-      disable_pmem
-    fi
     if [[ ! "$architecture" =~ amd64|arm64|armhf|ppc64el|s390x ]]
     then
       replace_uring_with_aio

debian/control

@@ -5,9 +5,9 @@ Maintainer: MariaDB Developers <developers@lists.mariadb.org>
 Build-Depends: bison,
                cmake,
                cracklib-runtime <!nocheck>,
-               debhelper (>= 10),
+               debhelper (>= 11),
-               dh-exec,
                default-jdk,
+               dh-exec,
                flex [amd64],
                gdb <!nocheck>,
                libaio-dev [linux-any],
@@ -35,7 +35,6 @@ Build-Depends: bison,
                libnuma-dev [linux-any],
                libpam0g-dev,
                libpcre2-dev,
-               libpmem-dev [amd64 arm64 ppc64el riscv64],
                libsnappy-dev,
                libssl-dev,
                libssl-dev:native,
@@ -594,20 +593,6 @@ Conflicts: mariadb-client (<< 11.0.0),
            mariadb-client-core-5.2,
            mariadb-client-core-5.3,
            mariadb-client-core-5.5,
-           mysql-client (<< 5.0.51),
-           mysql-client (<< 5.0.51),
-           mysql-client-5.0,
-           mysql-client-5.1,
-           mysql-client-5.5,
-           mysql-client-5.6,
-           mysql-client-5.7,
-           mysql-client-8.0,
-           mysql-client-core-5.0,
-           mysql-client-core-5.1,
-           mysql-client-core-5.5,
-           mysql-client-core-5.6,
-           mysql-client-core-5.7,
-           mysql-client-core-8.0,
            mariadb-server (<< 11.0.0),
            mariadb-server-10.0,
            mariadb-server-10.1,
@@ -628,13 +613,26 @@ Conflicts: mariadb-client (<< 11.0.0),
            mariadb-server-core-10.6,
            mariadb-server-core-10.7,
            mariadb-server-core-10.8,
-           mysql-server-core-8.0,
+           mysql-client (<< 5.0.51),
+           mysql-client-5.0,
+           mysql-client-5.1,
+           mysql-client-5.5,
+           mysql-client-5.6,
+           mysql-client-5.7,
+           mysql-client-8.0,
+           mysql-client-core-5.0,
+           mysql-client-core-5.1,
+           mysql-client-core-5.5,
+           mysql-client-core-5.6,
+           mysql-client-core-5.7,
+           mysql-client-core-8.0,
            mysql-server-5.7,
-           percona-server-server-5.6,
+           mysql-server-core-8.0,
            percona-server-server,
+           percona-server-server-5.6,
+           percona-xtradb-cluster-server,
            percona-xtradb-cluster-server-5.6,
-           percona-xtradb-cluster-server-5.7,
+           percona-xtradb-cluster-server-5.7
-           percona-xtradb-cluster-server
 Multi-Arch: foreign
 Description: MySQL compatibility links to mariadb-client binaries/scripts.
  The package contains links and binaries that are needed by MySQL centric
@@ -759,8 +757,8 @@ Suggests: mailx,
           mariadb-test,
           netcat-openbsd
 Recommends: libhtml-template-perl,
-            pv,
+            mariadb-server-compat,
-            mariadb-server-compat
+            pv
 Pre-Depends: adduser (>= 3.40),
              debconf,
              mariadb-common (>= ${source:Version})
@@ -865,22 +863,8 @@ Description: MariaDB database server binaries
 
 Package: mariadb-server-compat
 Architecture: all
-Depends: mariadb-server (>= ${source:Version}),
+Depends: mariadb-server (>= ${source:Version})
-Conflicts: mariadb-server-core (<< 11.0.0),
+Conflicts: mariadb-server (<< 11.0.0),
-           mariadb-server-core-10.0,
-           mariadb-server-core-10.1,
-           mariadb-server-core-10.2,
-           mariadb-server-core-10.3,
-           mariadb-server-core-10.4,
-           mariadb-server-core-10.5,
-           mariadb-server-core-10.6,
-           mariadb-server-core-10.7,
-           mariadb-server-core-10.8,
-           mariadb-server-core-5.1,
-           mariadb-server-core-5.2,
-           mariadb-server-core-5.3,
-           mariadb-server-core-5.5,
-           mariadb-server (<< 11.0.0),
            mariadb-server-10.0,
            mariadb-server-10.1,
            mariadb-server-10.2,
@@ -894,6 +878,20 @@ Conflicts: mariadb-server-core (<< 11.0.0),
            mariadb-server-5.2,
            mariadb-server-5.3,
            mariadb-server-5.5,
+           mariadb-server-core (<< 11.0.0),
+           mariadb-server-core-10.0,
+           mariadb-server-core-10.1,
+           mariadb-server-core-10.2,
+           mariadb-server-core-10.3,
+           mariadb-server-core-10.4,
+           mariadb-server-core-10.5,
+           mariadb-server-core-10.6,
+           mariadb-server-core-10.7,
+           mariadb-server-core-10.8,
+           mariadb-server-core-5.1,
+           mariadb-server-core-5.2,
+           mariadb-server-core-5.3,
+           mariadb-server-core-5.5,
            mysql-server-5.0,
            mysql-server-core-5.0,
            mysql-server-core-5.1,
@@ -901,12 +899,12 @@ Conflicts: mariadb-server-core (<< 11.0.0),
            mysql-server-core-5.6,
            mysql-server-core-5.7,
            mysql-server-core-8.0,
+           percona-server-server,
            percona-server-server-5.6,
            percona-server-server-5.7,
-           percona-server-server,
+           percona-xtradb-cluster-server,
            percona-xtradb-cluster-server-5.6,
-           percona-xtradb-cluster-server-5.7,
+           percona-xtradb-cluster-server-5.7
-           percona-xtradb-cluster-server
 Multi-Arch: foreign
 Description: MySQL compatibility links to mariadb-server binaries/scripts.
  The package contains links and binaries that are needed by MySQL centric
@@ -967,10 +965,10 @@ Description: Connect storage engine for MariaDB server
 Package: mariadb-plugin-connect-jdbc
 Architecture: any
 Depends: default-jre-headless
-Suggests: libmariadb-java,
+Suggests: libcsvjdbc-java,
-          libcsvjdbc-java,
+          libmariadb-java,
-          libpostgresql-jdbc-java,
           libpostgis-java,
+          libpostgresql-jdbc-java,
           libxerial-sqlite-jdbc-java
 Enhances: mariadb-plugin-connect
 Description: Connect storage engine JDBC interface for MariaDB server.
@@ -1214,11 +1212,11 @@ Description: Snappy compression support in the server and storage engines
 
 Package: mariadb-test
 Architecture: any
-Depends: mariadb-client (= ${binary:Version}),
+Depends: libnet-ssleay-perl,
+         mariadb-client (= ${binary:Version}),
          mariadb-server (= ${server:Version}),
          mariadb-test-data (= ${source:Version}),
          virtual-mysql-testsuite,
-         libnet-ssleay-perl,
          ${misc:Depends},
          ${perl:Depends},
          ${shlibs:Depends}

debian/mariadb-client-compat.install

@@ -1,20 +1,20 @@
 usr/bin/mysql
-usr/bin/mysqlbinlog
 usr/bin/mysql_convert_table_format
 usr/bin/mysql_find_rows
 usr/bin/mysql_fix_extensions
 usr/bin/mysql_plugin
+usr/bin/mysql_secure_installation
 usr/bin/mysql_setpermission
 usr/bin/mysql_tzinfo_to_sql
 usr/bin/mysql_waitpid
 usr/bin/mysqlaccess
 usr/bin/mysqladmin
+usr/bin/mysqlbinlog
 usr/bin/mysqlcheck
 usr/bin/mysqldump
 usr/bin/mysqldumpslow
 usr/bin/mysqlhotcopy
 usr/bin/mysqlimport
-usr/bin/mysql_secure_installation
 usr/bin/mysqlshow
 usr/bin/mysqlslap
 usr/share/man/man1/mysql.1
@@ -29,10 +29,10 @@ usr/share/man/man1/mysql_waitpid.1
 usr/share/man/man1/mysqlaccess.1
 usr/share/man/man1/mysqladmin.1
 usr/share/man/man1/mysqlbinlog.1
+usr/share/man/man1/mysqlcheck.1
 usr/share/man/man1/mysqldump.1
 usr/share/man/man1/mysqldumpslow.1
 usr/share/man/man1/mysqlhotcopy.1
 usr/share/man/man1/mysqlimport.1
 usr/share/man/man1/mysqlshow.1
 usr/share/man/man1/mysqlslap.1
-usr/share/man/man1/mysqlcheck.1

debian/mariadb-server-compat.install

@@ -1,12 +1,12 @@
+usr/bin/mysql_install_db
+usr/bin/mysql_upgrade
 usr/bin/mysqld_multi
 usr/bin/mysqld_safe
 usr/bin/mysqld_safe_helper
-usr/bin/mysql_install_db
-usr/bin/mysql_upgrade
 usr/sbin/mysqld
+usr/share/man/man1/mysql_install_db.1
+usr/share/man/man1/mysql_upgrade.1
 usr/share/man/man1/mysqld_multi.1
 usr/share/man/man1/mysqld_safe.1
 usr/share/man/man1/mysqld_safe_helper.1
-usr/share/man/man1/mysql_install_db.1
-usr/share/man/man1/mysql_upgrade.1
 usr/share/man/man8/mysqld.8

debian/mariadb-server-core.install

@@ -9,8 +9,8 @@ usr/share/man/man1/mariadb-upgrade.1
 usr/share/man/man1/resolveip.1
 usr/share/man/man8/mariadbd.8
 usr/share/mariadb/bulgarian
-usr/share/mariadb/chinese
 usr/share/mariadb/charsets
+usr/share/mariadb/chinese
 usr/share/mariadb/czech
 usr/share/mariadb/danish
 usr/share/mariadb/dutch
@@ -18,8 +18,8 @@ usr/share/mariadb/english
 usr/share/mariadb/estonian
 usr/share/mariadb/fill_help_tables.sql
 usr/share/mariadb/french
-usr/share/mariadb/german
 usr/share/mariadb/georgian
+usr/share/mariadb/german
 usr/share/mariadb/greek
 usr/share/mariadb/hindi
 usr/share/mariadb/hungarian

debian/mariadb-server.install

@@ -8,6 +8,10 @@ etc/logrotate.d/mariadb
 etc/security/user_map.conf
 lib/*/security/pam_user_map.so
 lib/systemd/system/mariadb@bootstrap.service.d/use_galera_new_cluster.conf
+lib/systemd/system/mariadb-extra@.socket
+lib/systemd/system/mariadb.service
+lib/systemd/system/mariadb@.service
+lib/systemd/system/mariadb@.socket
 lib/systemd/system/mysql.service
 lib/systemd/system/mysqld.service
 support-files/rpm/enable_encryption.preset etc/mysql/mariadb.conf.d/99-enable-encryption.cnf.preset
@@ -69,6 +73,7 @@ usr/share/man/man1/myisam_ftdump.1
 usr/share/man/man1/myisamchk.1
 usr/share/man/man1/myisamlog.1
 usr/share/man/man1/myisampack.1
+usr/share/man/man1/wsrep_sst_backup.1
 usr/share/man/man1/wsrep_sst_common.1
 usr/share/man/man1/wsrep_sst_mariabackup.1
 usr/share/man/man1/wsrep_sst_mysqldump.1

debian/mariadb-server.postinst

@@ -285,28 +285,5 @@ esac
 
 db_stop # in case invoke fails
 
-# dh_systemd_start doesn't emit anything since we still ship /etc/init.d/mariadb.
-# Thus MariaDB server is started via init.d script, which in turn redirects to
-# systemctl. If we upgrade from MySQL mysql.service may be masked, which also
-# means init.d script is disabled. Unmask mysql service explicitly.
-# Check first that the command exists, to avoid emitting any warning messages.
-if [ -x "$(command -v deb-systemd-helper)" ]
-then
-  deb-systemd-helper unmask mysql.service > /dev/null
-fi
-
 #DEBHELPER#
 
-# Modified dh_systemd_start snippet that's not added automatically
-if [ -d /run/systemd/system ]
-then
-  systemctl --system daemon-reload >/dev/null || true
-  deb-systemd-invoke start mariadb.service >/dev/null || true
-  # Modified dh_installinit snippet to only run with sysvinit
-elif [ -x "/etc/init.d/mariadb" ]
-then
-  if [ "$1" = "configure" ] || [ "$1" = "abort-upgrade" ]
-  then
-    invoke-rc.d mariadb start || exit $?
-  fi
-fi

debian/mariadb-server.postrm

@@ -12,50 +12,7 @@ fi
 
 ${DEBIAN_SCRIPT_TRACE:+ echo "#42#DEBUG# RUNNING $0 $*" 1>&2 }
 
-MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf"
+#DEBHELPER#
-
-# Try to stop the server in a sane way. If it does not success let the admin
-# do it himself. No database directories should be removed while the server
-# is running! Another mariadbd in e.g. a different chroot is fine for us.
-stop_server() {
-  # Return immediately if there are no mysqld processes running
-  # as there is no point in trying to shutdown in that case.
-  if ! pgrep -x --nslist pid --ns $$ "mysqld|mariadbd" > /dev/null
-  then
-    return
-  fi
-
-  set +e
-  invoke-rc.d mariadb stop
-  invoke-rc.d mysql stop # Backwards compatibility
-  errno=$?
-  set -e
-
-  # systemctl could emit exit code 100=no init script (fresh install)
-  if [ "$errno" != 0 ] && [ "$errno" != 100 ]
-  then
-    echo "Attempt to stop MariaDB/MySQL server returned exitcode $errno" 1>&2
-    echo "There is a MariaDB/MySQL server running, but we failed in our attempts to stop it." 1>&2
-    echo "Stop it yourself and try again!" 1>&2
-    db_stop
-    exit 1
-  fi
-}
-
-
-case "$1" in
-  purge|remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
-    if [ -n "$($MYADMIN ping 2>/dev/null)" ]
-    then
-      stop_server
-      sleep 2
-    fi
-    ;;
-  *)
-    echo "postrm called with unknown argument '$1'" 1>&2
-    exit 1
-    ;;
-esac
 
 #
 # - Purge logs and data only if they are ours (#307473)

debian/mariadb-server.prerm

@@ -1,14 +0,0 @@
-#!/bin/bash
-set -e
-
-#DEBHELPER#
-
-# Modified dh_systemd_start snippet that's not added automatically
-if [ -d /run/systemd/system ]
-then
-  deb-systemd-invoke stop mariadb.service >/dev/null
-  # Modified dh_installinit snippet to only run with sysvinit
-elif [ -x "/etc/init.d/mariadb" ]
-then
-  invoke-rc.d mariadb stop || exit $?
-fi

debian/not-installed

@@ -5,65 +5,61 @@ etc/mysql/mariadb.conf.d/enable_encryption.preset # Debian packaging uses files
 etc/mysql/mariadb.conf.d/mysql-clients.cnf # Debian packaging uses files from debian/additions/mariadb.cnf.d/
 etc/mysql/mariadb.conf.d/server.cnf # Debian packaging uses files from debian/additions/mariadb.cnf.d/
 lib/systemd/system/mariadb-extra.socket # Installed by rules file
-lib/systemd/system/mariadb-extra@.socket # Installed by rules file
-lib/systemd/system/mariadb.service # Installed by rules file
 lib/systemd/system/mariadb.socket # Installed by rules file
-lib/systemd/system/mariadb@.service # Installed by rules file
-lib/systemd/system/mariadb@.socket # Installed by rules file
 usr/bin/mariadb-embedded # Shipping the embedded server in distro packaging does not make sense
 usr/bin/mysql_config # Debian packaging has mysql_config as symlink to mariadb_config
 usr/bin/mysql_embedded # Symlink to mariadb-embedded which is intentionally not included
 usr/bin/sst_dump # Use the one from rocksdb-tools package
+usr/bin/uca-dump
+usr/bin/wsrep_sst_backup
 usr/lib/aarch64-linux-gnu/libdbbc.a # ColumnStore header file
 usr/lib/aarch64-linux-gnu/libidbboot.a # ColumnStore header file
 usr/lib/aarch64-linux-gnu/libprocessor.a # ColumnStore header file
 usr/lib/aarch64-linux-gnu/libwe_xml.a # ColumnStore header file
+usr/lib/sysusers.d/mariadb.conf # Not used (yet) in Debian systemd
+usr/lib/tmpfiles.d/mariadb.conf # Not used (yet) in Debian systemd
 usr/lib/x86_64-linux-gnu/libdbbc.a # ColumnStore header file
 usr/lib/x86_64-linux-gnu/libidbboot.a # ColumnStore header file
 usr/lib/x86_64-linux-gnu/libprocessor.a # ColumnStore header file
 usr/lib/x86_64-linux-gnu/libwe_xml.a # ColumnStore header file
-usr/bin/uca-dump
-usr/bin/wsrep_sst_backup
-usr/lib/sysusers.d/mariadb.conf # Not used (yet) in Debian systemd
-usr/lib/tmpfiles.d/mariadb.conf # Not used (yet) in Debian systemd
 usr/sbin/rcmysql
 usr/share/doc/mariadb-server/COPYING (related file: "debian/tmp/usr/share/mariadb/mroonga/COPYING")
 usr/share/doc/mariadb-server/CREDITS
 usr/share/doc/mariadb-server/INSTALL-BINARY
-usr/share/doc/mariadb-server/README-wsrep
 usr/share/doc/mariadb-server/README.md
+usr/share/doc/mariadb-server/README-wsrep
 usr/share/doc/mariadb-server/THIRDPARTY
-usr/share/groonga-normalizer-mysql/README.md
-usr/share/groonga-normalizer-mysql/lgpl-2.0.txt
 usr/share/groonga/COPYING
+usr/share/groonga-normalizer-mysql/lgpl-2.0.txt
+usr/share/groonga-normalizer-mysql/README.md
 usr/share/groonga/README.md
 usr/share/man/man1/mariadb-embedded.1 # Shipping the embedded server in distro packaging does not make sense
-usr/share/man/man1/mysql_embedded.1 # Symlink to mariadb-embedded.1 which is intentionally not included
 usr/share/man/man1/my_safe_process.1
-usr/share/man/man1/mysql-stress-test.pl.1
+usr/share/man/man1/mysql_embedded.1 # Symlink to mariadb-embedded.1 which is intentionally not included
 usr/share/man/man1/mysql.server.1
-usr/share/mariadb/Mongo2.jar
+usr/share/man/man1/mysql-stress-test.pl.1
-usr/share/mariadb/Mongo3.jar
 usr/share/mariadb/binary-configure
 usr/share/mariadb/magic
-usr/share/mariadb/mariadb.logrotate
 usr/share/mariadb/maria_add_gis_sp.sql # mariadb-server-core.install has *_bootstrap.sql
+usr/share/mariadb/mariadb.logrotate
 usr/share/mariadb/mariadb-test/asan.supp
 usr/share/mariadb/mariadb-test/lsan.supp
 usr/share/mariadb/mariadb-test/unstable-tests
-usr/share/mariadb/mysql.server # Debian packaging uses mariadb-server.mariadb.init
+usr/share/mariadb/Mongo2.jar
+usr/share/mariadb/Mongo3.jar
 usr/share/mariadb/mysqld_multi.server
+usr/share/mariadb/mysql.server # Debian packaging uses mariadb-server.mariadb.init
 usr/share/mariadb/policy/apparmor/README # In MariaDB we don't want to use AppArmor at the moment
 usr/share/mariadb/policy/apparmor/usr.sbin.mysqld # In MariaDB we don't want to use AppArmor at the moment
 usr/share/mariadb/policy/apparmor/usr.sbin.mysqld.local # In MariaDB we don't want to use AppArmor at the moment
-usr/share/mariadb/policy/selinux/README # In MariaDB we don't want to use SELinux at the moment
 usr/share/mariadb/policy/selinux/mariadb-server.fc # In MariaDB we don't want to use SELinux at the moment
 usr/share/mariadb/policy/selinux/mariadb-server.te # In MariaDB we don't want to use SELinux at the moment
 usr/share/mariadb/policy/selinux/mariadb.te # In MariaDB we don't want to use SELinux at the moment
+usr/share/mariadb/policy/selinux/README # In MariaDB we don't want to use SELinux at the moment
 usr/share/mariadb/systemd/mariadb-extra@.socket # Installed by rules file
 usr/share/mariadb/systemd/mariadb.service # Installed by rules file
 usr/share/mariadb/systemd/mariadb@.service # Installed by rules file
 usr/share/mariadb/systemd/mariadb@.socket # Installed by rules file
-usr/share/mariadb/systemd/mysql.service  # Installed by rules file
 usr/share/mariadb/systemd/mysqld.service  # Installed by rules file
+usr/share/mariadb/systemd/mysql.service  # Installed by rules file
 usr/share/mariadb/systemd/use_galera_new_cluster.conf

debian/rules

@@ -24,7 +24,7 @@ ifneq (,$(filter linux,$(DEB_HOST_ARCH_OS)))
 endif
 
 BUILDDIR := builddir
-DEB_VERSION_REVISION := $(shell echo $(DEB_VERSION) | sed -e 's/^.*-//')
+DEB_VERSION_REVISION := $(shell echo $(DEB_VERSION) | sed -e 's/.*[~-]\(.*\)/\1/')
 DEB_VERSION_VERSION := $(shell echo $(DEB_VERSION) | sed -e 's/^.*:\(.*\)\(-\|+\).*/\1/')
 DEB_VERSION_MAJOR := $(shell echo $(DEB_VERSION_VERSION) | sed -e 's/^\(.*\)\..*$$/\1/')
 RELEASE := $(shell lsb_release -r -s) # Use changelog based DEB_DISTRIBUTION instead?
@@ -51,12 +51,6 @@ ifeq (32,$(DEB_HOST_ARCH_BITS))
     CMAKEFLAGS += -DPLUGIN_ROCKSDB=NO
 endif
 
-# Only attempt to build with PMEM on archs that have package libpmem-dev available
-# See https://packages.debian.org/search?searchon=names&keywords=libpmem-dev
-ifneq (,$(filter $(DEB_HOST_ARCH),amd64 arm64 ppc64el riscv64))
-    CMAKEFLAGS += -DWITH_PMEM=ON
-endif
-
 # Add support for verbose builds
 MAKEFLAGS += VERBOSE=1
 
@@ -98,7 +92,6 @@ endif
 	    -DCOMPILATION_COMMENT="mariadb.org binary distribution" \
 	    -DMYSQL_SERVER_SUFFIX="-$(DEB_VERSION_REVISION)" \
 	    -DSYSTEM_TYPE="debian-$(DEB_HOST_GNU_SYSTEM)" \
-	    -DCMAKE_SYSTEM_PROCESSOR=$(DEB_HOST_ARCH) \
 	    -DBUILD_CONFIG=mysql_release \
 	    -DCONC_DEFAULT_CHARSET=utf8mb4 \
 	    -DPLUGIN_AWS_KEY_MANAGEMENT=NO \
@@ -133,16 +126,6 @@ override_dh_auto_install:
 	dh_testdir
 	dh_testroot
 
-ifneq (,$(filter linux,$(DEB_HOST_ARCH_OS)))
-	# Copy systemd files to a location available for dh_installinit
-	cp $(BUILDDIR)/support-files/mariadb.service debian/mariadb-server.mariadb.service
-	cp $(BUILDDIR)/support-files/mariadb.socket debian/mariadb-server.mariadb.socket
-	cp $(BUILDDIR)/support-files/mariadb-extra.socket debian/mariadb-server.mariadb-extra.socket
-	cp $(BUILDDIR)/support-files/mariadb@.service debian/mariadb-server.mariadb@.service
-	cp $(BUILDDIR)/support-files/mariadb@.socket debian/mariadb-server.mariadb@.socket
-	cp $(BUILDDIR)/support-files/mariadb-extra@.socket debian/mariadb-server.mariadb-extra@.socket
-endif
-
 	# Run 'make install' without output since it is uninteresting and
 	# silencing it helps to make overall build log shorter and more readable
 	@echo "Running $(MAKE) install DESTDIR=$(TMP) ..."
@@ -185,8 +168,8 @@ endif
 	# Move test plugins that are only needed by the client to the libmariadb path
 	mv -v $(TMP)/usr/lib/mysql/plugin/qa_auth_client.so $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb3/plugin/
 
-override_dh_systemd_enable:
+override_dh_installsystemd:
-	dh_systemd_enable --name=mariadb
+	dh_systemd_enable --name=mariadb mariadb.service
 	dh_systemd_enable --no-enable --name=mariadb mariadb.socket
 	dh_systemd_enable --no-enable --name=mariadb-extra mariadb-extra.socket
 	dh_systemd_enable --no-enable --name=mariadb@ mariadb.socket
@@ -196,7 +179,7 @@ override_dh_systemd_enable:
 # Start MariaDB at sequence number 19 before 20 where apache, proftpd etc gets
 # started which might depend on a running database server.
 override_dh_installinit-arch:
-	dh_installinit --name=mariadb --no-start -- defaults 19 21
+	dh_installinit --name=mariadb -- defaults 19 21
 	dh_systemd_start --restart-after-upgrade
 
 # Use custom server version string variable

debian/tests/control

@@ -3,14 +3,15 @@ Tests: smoke
 # for its existence (see the list in debian/control), install it if available
 # and check in the test if it's functional when it should be.
 # The plugin package also already depends on the other one.
-Depends: mariadb-plugin-rocksdb | mariadb-server,
+Depends: mariadb-plugin-provider-bzip2,
-         mariadb-plugin-provider-bzip2,
          mariadb-plugin-provider-lz4,
          mariadb-plugin-provider-lzma,
          mariadb-plugin-provider-lzo,
-         mariadb-plugin-provider-snappy
+         mariadb-plugin-provider-snappy,
+         mariadb-plugin-rocksdb | mariadb-server
 Restrictions: allow-stderr needs-root isolation-container
 
 Tests: upstream
-Depends: mariadb-test, eatmydata
+Depends: eatmydata,
+         mariadb-test
 Restrictions: allow-stderr breaks-testbed

extra/CMakeLists.txt

@@ -13,7 +13,7 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1335 USA
 
-INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${ZLIB_INCLUDE_DIR})
+INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${ZLIB_INCLUDE_DIRS})
 
 # Default install component for the files is Server here
 SET(MYSQL_INSTALL_COMPONENT Server)

extra/mariabackup/CMakeLists.txt

@@ -31,6 +31,7 @@ ENDIF()
 INCLUDE_DIRECTORIES(
   ${CMAKE_SOURCE_DIR}/include
   ${CMAKE_SOURCE_DIR}/sql
+  ${CMAKE_SOURCE_DIR}/storage/maria
   ${CMAKE_CURRENT_SOURCE_DIR}/quicklz
   ${CMAKE_CURRENT_SOURCE_DIR}
   )
@@ -49,14 +50,9 @@ ADD_DEFINITIONS(-UMYSQL_SERVER)
 ADD_DEFINITIONS(-DPCRE_STATIC=1)
 ADD_DEFINITIONS(${SSL_DEFINES})
 
-IF(PMEM_FOUND)
-  ADD_COMPILE_FLAGS(xtrabackup.cc COMPILE_FLAGS "-DHAVE_PMEM")
-ENDIF()
-
 MYSQL_ADD_EXECUTABLE(mariadb-backup
   xtrabackup.cc
   innobackupex.cc
-  changed_page_bitmap.cc
   datasink.cc
   ds_buffer.cc
   ds_compress.cc
@@ -72,8 +68,12 @@ MYSQL_ADD_EXECUTABLE(mariadb-backup
   xbstream_write.cc
   backup_mysql.cc
   backup_copy.cc
-  xb_plugin.cc
+  encryption_plugin.cc
   ${PROJECT_BINARY_DIR}/sql/sql_builtin.cc
+  aria_backup_client.cc
+  thread_pool.cc
+  ddl_log.cc
+  common_engine.cc
   ${PROJECT_SOURCE_DIR}/sql/net_serv.cc
   ${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c
   COMPONENT Backup
@@ -82,7 +82,8 @@ MYSQL_ADD_EXECUTABLE(mariadb-backup
 # Export all symbols on Unix, for better crash callstacks
 SET_TARGET_PROPERTIES(mariadb-backup PROPERTIES ENABLE_EXPORTS TRUE)
 
-TARGET_LINK_LIBRARIES(mariadb-backup sql sql_builtins)
+TARGET_LINK_LIBRARIES(mariadb-backup  sql sql_builtins aria)
+
 IF(NOT HAVE_SYSTEM_REGEX)
   TARGET_LINK_LIBRARIES(mariadb-backup pcre2-posix)
 ENDIF()

extra/mariabackup/aria_backup_client.h

@@ -0,0 +1,38 @@
+#pragma once
+#include "my_global.h"
+#include "datasink.h"
+#include "backup_mysql.h"
+#include "thread_pool.h"
+#include "xtrabackup.h"
+
+namespace aria {
+
+bool prepare(const char *target_dir);
+
+class BackupImpl;
+
+class Backup {
+	public:
+		Backup(const char *datadir_path,
+		       const char *aria_log_path,
+		       ds_ctxt_t *datasink,
+			std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool);
+		~Backup();
+		Backup (Backup &&other) = delete;
+		Backup & operator= (Backup &&other) = delete;
+		Backup(const Backup &) = delete;
+		Backup & operator= (const Backup &) = delete;
+		bool init();
+		bool start(bool no_lock);
+		bool wait_for_finish();
+		bool copy_offline_tables(
+			const std::unordered_set<table_key_t> *exclude_tables, bool no_lock,
+			bool copy_stats);
+		bool finalize();
+		bool copy_log_tail();
+		void set_post_copy_table_hook(const post_copy_table_hook_t &hook);
+	private:
+		BackupImpl *m_backup_impl;
+};
+
+} // namespace aria

extra/mariabackup/backup_copy.cc

@@ -41,6 +41,9 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA
 *******************************************************/
 
 #include <my_global.h>
+#include <my_config.h>
+#include <unireg.h>
+#include <datadict.h>
 #include <os0file.h>
 #include <my_dir.h>
 #include <ut0mem.h>
@@ -66,19 +69,26 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA
 #include <aclapi.h>
 #endif
 
+#ifdef MYSQL_CLIENT
+#define WAS_MYSQL_CLIENT 1
+#undef MYSQL_CLIENT
+#endif
+
+#include "table.h"
+
+#ifdef WAS_MYSQL_CLIENT
+#define MYSQL_CLIENT 1
+#undef WAS_MYSQL_CLIENT
+#endif
 
 #define ROCKSDB_BACKUP_DIR "#rocksdb"
 
-/* list of files to sync for --rsync mode */
-static std::set<std::string> rsync_list;
 /* locations of tablespaces read from .isl files */
 static std::map<std::string, std::string> tablespace_locations;
 
 /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
 bool binlog_locked;
 
-static void rocksdb_create_checkpoint();
-static bool has_rocksdb_plugin();
 static void rocksdb_backup_checkpoint(ds_ctxt *ds_data);
 static void rocksdb_copy_back(ds_ctxt *ds_data);
 
@@ -135,10 +145,6 @@ struct datadir_thread_ctxt_t {
 	bool			ret;
 };
 
-static bool backup_files_from_datadir(ds_ctxt_t *ds_data,
-                                      const char *dir_path,
-                                      const char *prefix);
-
 /************************************************************************
 Retirn true if character if file separator */
 bool
@@ -585,7 +591,6 @@ datafile_read(datafile_cur_t *cursor)
 Check to see if a file exists.
 Takes name of the file to check.
 @return true if file exists. */
-static
 bool
 file_exists(const char *filename)
 {
@@ -601,7 +606,6 @@ file_exists(const char *filename)
 
 /************************************************************************
 Trim leading slashes from absolute path so it becomes relative */
-static
 const char *
 trim_dotslash(const char *path)
 {
@@ -634,7 +638,7 @@ ends_with(const char *str, const char *suffix)
 	       && strcmp(str + str_len - suffix_len, suffix) == 0);
 }
 
-static bool starts_with(const char *str, const char *prefix)
+bool starts_with(const char *str, const char *prefix)
 {
 	return strncmp(str, prefix, strlen(prefix)) == 0;
 }
@@ -785,7 +789,6 @@ directory_exists_and_empty(const char *dir, const char *comment)
 /************************************************************************
 Check if file name ends with given set of suffixes.
 @return true if it does. */
-static
 bool
 filename_matches(const char *filename, const char **ext_list)
 {
@@ -800,6 +803,115 @@ filename_matches(const char *filename, const char **ext_list)
 	return(false);
 }
 
+// TODO: the code can be used to find storage engine of partitions
+/*
+static
+bool is_aria_frm_or_par(const char *path) {
+	if (!ends_with(path, ".frm") && !ends_with(path, ".par"))
+		return false;
+
+	const char *frm_path = path;
+	if (ends_with(path, ".par")) {
+		size_t frm_path_len = strlen(path);
+		DBUG_ASSERT(frm_path_len > strlen("frm"));
+		frm_path = strdup(path);
+		strcpy(const_cast<char *>(frm_path) + frm_path_len - strlen("frm"), "frm");
+	}
+
+	bool result = false;
+	File file;
+	uchar header[40];
+	legacy_db_type dbt;
+
+	if ((file= mysql_file_open(key_file_frm, frm_path, O_RDONLY | O_SHARE, MYF(0)))
+			< 0)
+		goto err;
+
+	if (mysql_file_read(file, (uchar*) header, sizeof(header), MYF(MY_NABP)))
+		goto err;
+
+	if (!strncmp((char*) header, "TYPE=VIEW\n", 10))
+		goto err;
+
+	if (!is_binary_frm_header(header))
+		goto err;
+
+	dbt = (legacy_db_type)header[3];
+
+	if (dbt == DB_TYPE_ARIA) {
+		result = true;
+	}
+	else if (dbt == DB_TYPE_PARTITION_DB) {
+		MY_STAT state;
+		uchar *frm_image= 0;
+//		uint n_length;
+
+		if (mysql_file_fstat(file, &state, MYF(MY_WME)))
+			goto err;
+
+		if (mysql_file_seek(file, 0, SEEK_SET, MYF(MY_WME)))
+			goto err;
+
+		if (read_string(file, &frm_image, (size_t)state.st_size))
+			goto err;
+
+		dbt = (legacy_db_type)frm_image[61];
+		if (dbt == DB_TYPE_ARIA) {
+			result = true;
+		}
+		my_free(frm_image);
+	}
+
+err:
+	if (file >= 0)
+		mysql_file_close(file, MYF(MY_WME));
+	if (frm_path != path)
+		free(const_cast<char *>(frm_path));
+  return result;
+}
+*/
+
+void parse_db_table_from_file_path(
+	const char *filepath, char *dbname, char *tablename) {
+	dbname[0] = '\0';
+	tablename[0] = '\0';
+	const char *dbname_start = nullptr;
+	const char *tablename_start = filepath;
+	const char *const_ptr;
+	while ((const_ptr = strchr(tablename_start, FN_LIBCHAR)) != NULL) {
+		dbname_start = tablename_start;
+		tablename_start = const_ptr + 1;
+	}
+	if (!dbname_start)
+		return;
+	size_t dbname_len = tablename_start - dbname_start - 1;
+	if (dbname_len >= FN_REFLEN)
+          dbname_len = FN_REFLEN-1;
+	strmake(dbname, dbname_start, dbname_len);
+	strmake(tablename, tablename_start, FN_REFLEN-1);
+	char *ptr;
+	if ((ptr = strchr(tablename, '.')))
+		*ptr = '\0';
+	if ((ptr = strstr(tablename, "#P#")))
+		*ptr = '\0';
+}
+
+bool is_system_table(const char *dbname, const char *tablename)
+{
+	DBUG_ASSERT(dbname);
+	DBUG_ASSERT(tablename);
+
+	Lex_ident_db lex_dbname;
+	Lex_ident_table lex_tablename;
+	lex_dbname.str = dbname;
+	lex_dbname.length = strlen(dbname);
+	lex_tablename.str = tablename;
+	lex_tablename.length = strlen(tablename);
+
+	TABLE_CATEGORY tg = get_table_category(lex_dbname, lex_tablename);
+
+	return (tg == TABLE_CATEGORY_LOG) || (tg == TABLE_CATEGORY_SYSTEM);
+}
 
 /************************************************************************
 Copy data file for backup. Also check if it is allowed to copy by
@@ -810,8 +922,7 @@ static
 bool
 datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n)
 {
-	const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
+       const char *ext_list[] = {".frm", ".isl", ".TRG", ".TRN", ".opt", ".par",
-		"MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
          NULL};
 
 	/* Get the name and the path for the tablespace. node->name always
@@ -835,41 +946,6 @@ datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n)
 	return(true);
 }
 
-
-/************************************************************************
-Same as datafile_copy_backup, but put file name into the list for
-rsync command. */
-static
-bool
-datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f)
-{
-	const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
-		"MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
-		NULL};
-
-	/* Get the name and the path for the tablespace. node->name always
-	contains the path (which may be absolute for remote tablespaces in
-	5.6+). space->name contains the tablespace name in the form
-	"./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
-	multi-node shared tablespace, space->name contains the name of the first
-	node, but that's irrelevant, since we only need node_name to match them
-	against filters, and the shared tablespace is always copied regardless
-	of the filters value. */
-
-	if (check_if_skip_table(filepath)) {
-		return(true);
-	}
-
-	if (filename_matches(filepath, ext_list)) {
-		fprintf(f, "%s\n", filepath);
-		if (save_to_list) {
-			rsync_list.insert(filepath);
-		}
-	}
-
-	return(true);
-}
-
 bool ds_ctxt_t::backup_file_print_buf(const char *filename,
                                       const char *buf, int buf_len)
 {
@@ -1004,16 +1080,15 @@ Copy file for backup/restore.
 bool
 ds_ctxt_t::copy_file(const char *src_file_path,
                      const char *dst_file_path,
-                     uint thread_n)
+		     uint thread_n,
+		     bool rewrite)
 {
 	char			 dst_name[FN_REFLEN];
 	ds_file_t		*dstfile = NULL;
 	datafile_cur_t		 cursor;
 	xb_fil_cur_result_t	 res;
 	DBUG_ASSERT(datasink->remove);
-	const char	*dst_path =
+	const char	*dst_path = convert_dst(dst_file_path);
-		(xtrabackup_copy_back || xtrabackup_move_back)?
-		dst_file_path : trim_dotslash(dst_file_path);
 
 	if (!datafile_open(src_file_path, &cursor, thread_n)) {
 		goto error_close;
@@ -1021,7 +1096,7 @@ ds_ctxt_t::copy_file(const char *src_file_path,
 
 	strncpy(dst_name, cursor.rel_path, sizeof(dst_name));
 
-	dstfile = ds_open(this, dst_path, &cursor.statinfo);
+	dstfile = ds_open(this, dst_path, &cursor.statinfo, rewrite);
 	if (dstfile == NULL) {
 		msg(thread_n,"error: "
 			"cannot open the destination stream for %s", dst_name);
@@ -1245,278 +1320,45 @@ cleanup:
 }
 
 
-
-
-static
 bool
-backup_files(ds_ctxt *ds_data, const char *from, bool prep_mode)
+backup_files(ds_ctxt *ds_data, const char *from)
 {
-	char rsync_tmpfile_name[FN_REFLEN];
-	FILE *rsync_tmpfile = NULL;
 	datadir_iter_t *it;
 	datadir_node_t node;
 	bool ret = true;
-
+	msg("Starting to backup non-InnoDB tables and files");
-	if (prep_mode && !opt_rsync) {
-		return(true);
-	}
-
-	if (opt_rsync) {
-		snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
-			"%s/%s%d", opt_mysql_tmpdir,
-			"xtrabackup_rsyncfiles_pass",
-			prep_mode ? 1 : 2);
-		rsync_tmpfile = fopen(rsync_tmpfile_name, "w");
-		if (rsync_tmpfile == NULL) {
-			msg("Error: can't create file %s",
-				rsync_tmpfile_name);
-			return(false);
-		}
-	}
-
-	msg("Starting %s non-InnoDB tables and files",
-	       prep_mode ? "prep copy of" : "to backup");
-
 	datadir_node_init(&node);
 	it = datadir_iter_new(from);
-
 	while (datadir_iter_next(it, &node)) {
-
 		if (!node.is_empty_dir) {
-			if (opt_rsync) {
-				ret = datafile_rsync_backup(node.filepath,
-					!prep_mode, rsync_tmpfile);
-			} else {
 			ret = datafile_copy_backup(ds_data, node.filepath, 1);
-			}
 			if (!ret) {
 				msg("Failed to copy file %s", node.filepath);
 				goto out;
 			}
-		} else if (!prep_mode) {
+		} else {
 			/* backup fake file into empty directory */
 			char path[FN_REFLEN];
-			snprintf(path, sizeof(path),
+			snprintf(path, sizeof(path), "%s/db.opt", node.filepath);
-				 "%s/db.opt", node.filepath);
+			if (!(ret = ds_data->backup_file_printf(trim_dotslash(path), "%s", ""))) {
-			if (!(ret = ds_data->backup_file_printf(
-					trim_dotslash(path), "%s", ""))) {
 				msg("Failed to create file %s", path);
 				goto out;
 			}
 		}
 	}
-
+	msg("Finished backing up non-InnoDB tables and files");
-	if (opt_rsync) {
-		std::stringstream cmd;
-		int err;
-
-		if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
-			fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename);
-			rsync_list.insert(buffer_pool_filename);
-		}
-		if (file_exists("ib_lru_dump")) {
-			fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump");
-			rsync_list.insert("ib_lru_dump");
-		}
-
-		fclose(rsync_tmpfile);
-		rsync_tmpfile = NULL;
-
-		cmd << "rsync -t . --files-from=" << rsync_tmpfile_name
-		    << " " << xtrabackup_target_dir;
-
-		msg("Starting rsync as: %s", cmd.str().c_str());
-		if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) {
-			msg("Error: rsync failed with error code %d", err);
-			ret = false;
-			goto out;
-		}
-		msg("rsync finished successfully.");
-
-		if (!prep_mode && !opt_no_lock) {
-			char path[FN_REFLEN];
-			char dst_path[FN_REFLEN];
-			char *newline;
-
-			/* Remove files that have been removed between first and
-			second passes. Cannot use "rsync --delete" because it
-			does not work with --files-from. */
-			snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
-				"%s/%s", opt_mysql_tmpdir,
-				"xtrabackup_rsyncfiles_pass1");
-
-			rsync_tmpfile = fopen(rsync_tmpfile_name, "r");
-			if (rsync_tmpfile == NULL) {
-				msg("Error: can't open file %s",
-					rsync_tmpfile_name);
-				ret = false;
-				goto out;
-			}
-
-			while (fgets(path, sizeof(path), rsync_tmpfile)) {
-
-				newline = strchr(path, '\n');
-				if (newline) {
-					*newline = 0;
-				}
-				if (rsync_list.count(path) < 1) {
-					snprintf(dst_path, sizeof(dst_path),
-						"%s/%s", xtrabackup_target_dir,
-						path);
-					msg("Removing %s", dst_path);
-					unlink(dst_path);
-				}
-			}
-
-			fclose(rsync_tmpfile);
-			rsync_tmpfile = NULL;
-		}
-	}
-
-	msg("Finished %s non-InnoDB tables and files",
-	       prep_mode ? "a prep copy of" : "backing up");
-
 out:
 	datadir_iter_free(it);
 	datadir_node_free(&node);
-
-	if (rsync_tmpfile != NULL) {
-		fclose(rsync_tmpfile);
-	}
-
 	return(ret);
 }
 
-
-lsn_t get_current_lsn(MYSQL *connection)
-{
-	static const char lsn_prefix[] = "\nLog sequence number ";
-	lsn_t lsn = 0;
-	if (MYSQL_RES *res = xb_mysql_query(connection,
-					    "SHOW ENGINE INNODB STATUS",
-					    true, false)) {
-		if (MYSQL_ROW row = mysql_fetch_row(res)) {
-			const char *p= strstr(row[2], lsn_prefix);
-			DBUG_ASSERT(p);
-			if (p) {
-				p += sizeof lsn_prefix - 1;
-				lsn = lsn_t(strtoll(p, NULL, 10));
-			}
-		}
-		mysql_free_result(res);
-	}
-	return lsn;
-}
-
 lsn_t server_lsn_after_lock;
 extern void backup_wait_for_lsn(lsn_t lsn);
-/** Start --backup */
-bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta,
-                  CorruptedPages &corrupted_pages)
-{
-	if (!opt_no_lock) {
-		if (opt_safe_slave_backup) {
-			if (!wait_for_safe_slave(mysql_connection)) {
-				return(false);
-			}
-		}
 
-		if (!backup_files(ds_data, fil_path_to_mysql_datadir, true)) {
+/** Release resources after backup_files() */
-			return(false);
-		}
-
-		history_lock_time = time(NULL);
-
-		if (!lock_tables(mysql_connection)) {
-			return(false);
-		}
-		server_lsn_after_lock = get_current_lsn(mysql_connection);
-	}
-
-	if (!backup_files(ds_data, fil_path_to_mysql_datadir, false)) {
-		return(false);
-	}
-
-        if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir,
-	                               "aws-kms-key") ||
-            !backup_files_from_datadir(ds_data,
-                                       aria_log_dir_path,
-                                       "aria_log")) {
-		return false;
-	}
-
-	if (has_rocksdb_plugin()) {
-		rocksdb_create_checkpoint();
-	}
-
-	msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock);
-	backup_wait_for_lsn(server_lsn_after_lock);
-	DBUG_EXECUTE_FOR_KEY("sleep_after_waiting_for_lsn", {},
-		{
-			ulong milliseconds = strtoul(dbug_val, NULL, 10);
-			msg("sleep_after_waiting_for_lsn");
-			my_sleep(milliseconds*1000UL);
-		});
-
-	corrupted_pages.backup_fix_ddl(ds_data, ds_meta);
-
-	// There is no need to stop slave thread before coping non-Innodb data when
-	// --no-lock option is used because --no-lock option requires that no DDL or
-	// DML to non-transaction tables can occur.
-	if (opt_no_lock) {
-		if (opt_safe_slave_backup) {
-			if (!wait_for_safe_slave(mysql_connection)) {
-				return(false);
-			}
-		}
-	}
-
-	if (opt_slave_info) {
-		lock_binlog_maybe(mysql_connection);
-
-		if (!write_slave_info(ds_data, mysql_connection)) {
-			return(false);
-		}
-	}
-
-	/* The only reason why Galera/binlog info is written before
-	wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup
-	binary will start streamig a temporary copy of REDO log to stdout and
-	thus, any streaming from innobackupex would interfere. The only way to
-	avoid that is to have a single process, i.e. merge innobackupex and
-	xtrabackup. */
-	if (opt_galera_info) {
-		if (!write_galera_info(ds_data, mysql_connection)) {
-			return(false);
-		}
-	}
-
-	if (opt_binlog_info == BINLOG_INFO_ON) {
-
-		lock_binlog_maybe(mysql_connection);
-		write_binlog_info(ds_data, mysql_connection);
-	}
-
-	if (!opt_no_lock) {
-		msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS...");
-		xb_mysql_query(mysql_connection,
-			"FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false);
-	}
-
-	return(true);
-}
-
-/** Release resources after backup_start() */
 void backup_release()
 {
-	/* release all locks */
-	if (!opt_no_lock) {
-		unlock_all(mysql_connection);
-		history_lock_time = 0;
-	} else {
-		history_lock_time = time(NULL) - history_lock_time;
-	}
-
 	if (opt_lock_ddl_per_table) {
 		mdl_unlock_all();
 	}
@@ -1530,11 +1372,11 @@ void backup_release()
 
 static const char *default_buffer_pool_file = "ib_buffer_pool";
 
-/** Finish after backup_start() and backup_release() */
+/** Finish after backup_files() and backup_release() */
 bool backup_finish(ds_ctxt *ds_data)
 {
 	/* Copy buffer pool dump or LRU dump */
-	if (!opt_rsync && opt_galera_info) {
+	if (opt_galera_info) {
 		if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
 			ds_data->copy_file(buffer_pool_filename, default_buffer_pool_file, 0);
 		}
@@ -1897,8 +1739,6 @@ copy_back()
 		return(false);
 	}
 
-	srv_max_n_threads = 1000;
-
 	/* copy undo tablespaces */
 
 	Copy_back_dst_dir dst_dir_buf;
@@ -1926,7 +1766,8 @@ copy_back()
 
 	dst_dir = dst_dir_buf.make(srv_log_group_home_dir);
 
-	/* --backup generates a single ib_logfile0, which we must copy. */
+	/* --backup generates a single LOG_FILE_NAME, which we must copy
+	if it exists. */
 
 	ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL);
 	if (!(ret = copy_or_move_file(ds_tmp, LOG_FILE_NAME, LOG_FILE_NAME,
@@ -2163,8 +2004,6 @@ decrypt_decompress()
 	bool ret;
 	datadir_iter_t *it = NULL;
 
-	srv_max_n_threads = 1000;
-
 	/* cd to backup directory */
 	if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
 	{
@@ -2177,8 +2016,6 @@ decrypt_decompress()
 
 	it = datadir_iter_new(".", false);
 
-	ut_a(xtrabackup_parallel >= 0);
-
 	ret = run_data_threads(it, decrypt_decompress_thread_func,
 		xtrabackup_parallel ? xtrabackup_parallel : 1);
 
@@ -2200,7 +2037,7 @@ decrypt_decompress()
   Do not copy the Innodb files (ibdata1, redo log files),
   as this is done in a separate step.
 */
-static bool backup_files_from_datadir(ds_ctxt_t *ds_data,
+bool backup_files_from_datadir(ds_ctxt_t *ds_data,
                                const char *dir_path,
                                const char *prefix)
 {
@@ -2226,10 +2063,6 @@ static bool backup_files_from_datadir(ds_ctxt_t *ds_data,
 			pname = info.name;
 
 		if (!starts_with(pname, prefix))
-			/* For ES exchange the above line with the following code:
-			(!xtrabackup_prepare || !xtrabackup_incremental_dir ||
-				!starts_with(pname, "aria_log")))
-			*/
 			continue;
 
 		if (xtrabackup_prepare && xtrabackup_incremental_dir &&
@@ -2252,7 +2085,7 @@ static int rocksdb_remove_checkpoint_directory()
 	return 0;
 }
 
-static bool has_rocksdb_plugin()
+bool has_rocksdb_plugin()
 {
 	static bool first_time = true;
 	static bool has_plugin= false;
@@ -2398,7 +2231,7 @@ static void rocksdb_unlock_checkpoint()
 #define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint"
 static 	char rocksdb_checkpoint_dir[FN_REFLEN];
 
-static void rocksdb_create_checkpoint()
+void rocksdb_create_checkpoint()
 {
 	MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true);
 	MYSQL_ROW row = mysql_fetch_row(result);
@@ -2478,3 +2311,39 @@ static void rocksdb_copy_back(ds_ctxt *ds_data) {
 	mkdirp(rocksdb_home_dir, 0777, MYF(0));
 	ds_data->copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back);
 }
+
+void foreach_file_in_db_dirs(
+	const char *dir_path, std::function<bool(const char *)> func) {
+	DBUG_ASSERT(dir_path);
+
+	datadir_iter_t *it;
+	datadir_node_t node;
+
+	datadir_node_init(&node);
+	it = datadir_iter_new(dir_path);
+
+	while (datadir_iter_next(it, &node))
+		if (!node.is_empty_dir && !func(node.filepath))
+			break;
+
+	datadir_iter_free(it);
+	datadir_node_free(&node);
+}
+
+void foreach_file_in_datadir(
+	const char *dir_path, std::function<bool(const char *)> func)
+{
+	DBUG_ASSERT(dir_path);
+	os_file_dir_t dir = os_file_opendir(dir_path);
+	os_file_stat_t info;
+	while (os_file_readdir_next_file(dir_path, dir, &info) == 0) {
+		if (info.type != OS_FILE_TYPE_FILE)
+			continue;
+		const char *pname = strrchr(info.name, IF_WIN('\\', '/'));
+		if (!pname)
+			pname = info.name;
+		if (!func(pname))
+			break;
+	}
+	os_file_closedir(dir);
+}

extra/mariabackup/backup_copy.h

@@ -2,6 +2,7 @@
 #ifndef XTRABACKUP_BACKUP_COPY_H
 #define XTRABACKUP_BACKUP_COPY_H
 
+#include <functional>
 #include <my_global.h>
 #include <mysql.h>
 #include "datasink.h"
@@ -29,11 +30,10 @@ bool
 equal_paths(const char *first, const char *second);
 
 /** Start --backup */
-bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta,
+bool backup_files(ds_ctxt *ds_data, const char *from);
-                  CorruptedPages &corrupted_pages);
+/** Release resources after backup_files() */
-/** Release resources after backup_start() */
 void backup_release();
-/** Finish after backup_start() and backup_release() */
+/** Finish after backup_files() and backup_release() */
 bool backup_finish(ds_ctxt *ds_data);
 bool
 apply_log_finish();
@@ -46,7 +46,25 @@ is_path_separator(char);
 bool
 directory_exists(const char *dir, bool create);
 
-lsn_t
+bool has_rocksdb_plugin();
-get_current_lsn(MYSQL *connection);
+void rocksdb_create_checkpoint();
+void foreach_file_in_db_dirs(
+	const char *dir_path, std::function<bool(const char *)> func);
+void foreach_file_in_datadir(
+	const char *dir_path, std::function<bool(const char *)> func);
+bool ends_with(const char *str, const char *suffix);
+bool starts_with(const char *str, const char *prefix);
+void parse_db_table_from_file_path(
+	const char *filepath, char *dbname, char *tablename);
+const char *trim_dotslash(const char *path);
+bool backup_files_from_datadir(ds_ctxt_t *ds_data,
+                               const char *dir_path,
+                               const char *prefix);
 
+bool is_system_table(const char *dbname, const char *tablename);
+std::unique_ptr<std::vector<std::string>>
+	find_files(const char *dir_path, const char *prefix, const char *suffix);
+bool file_exists(const char *filename);
+bool
+filename_matches(const char *filename, const char **ext_list);
 #endif

extra/mariabackup/backup_debug.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "my_dbug.h"
+
 #ifndef DBUG_OFF
 char *dbug_mariabackup_get_val(const char *event, fil_space_t::name_type key);
 /*
@@ -14,11 +15,21 @@ To use this facility, you need to
    for the variable)
 3. start mariabackup with --dbug=+d,debug_mariabackup_events
 */
+extern void dbug_mariabackup_event(
+	const char *event, const fil_space_t::name_type key, bool need_lock);
+#define DBUG_MARIABACKUP_EVENT(A, B) \
+	DBUG_EXECUTE_IF("mariabackup_events", \
+		dbug_mariabackup_event(A,B,false););
+#define DBUG_MARIABACKUP_EVENT_LOCK(A, B) \
+	DBUG_EXECUTE_IF("mariabackup_events", \
+		dbug_mariabackup_event(A,B, true););
 #define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \
-	DBUG_EXECUTE_IF("mariabackup_inject_code",		\
+	DBUG_EXECUTE_IF("mariabackup_inject_code", {\
-	{ char *dbug_val= dbug_mariabackup_get_val(EVENT, KEY);	\
+		char *dbug_val = dbug_mariabackup_get_val(EVENT, KEY); \
-	  if (dbug_val) CODE })
+		if (dbug_val && *dbug_val) CODE \
+	})
 #else
+#define DBUG_MARIABACKUP_EVENT(A,B)
+#define DBUG_MARIABACKUP_EVENT_LOCK(A,B)
 #define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE)
 #endif
-

extra/mariabackup/backup_mysql.cc

@@ -47,6 +47,12 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA
 #include <stdlib.h>
 #include <string.h>
 #include <limits>
+#ifdef HAVE_PWD_H
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#include <pwd.h>
+#endif
 #include "common.h"
 #include "xtrabackup.h"
 #include "srv0srv.h"
@@ -54,19 +60,19 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA
 #include "backup_copy.h"
 #include "backup_mysql.h"
 #include "mysqld.h"
-#include "xb_plugin.h"
+#include "encryption_plugin.h"
 #include <sstream>
 #include <sql_error.h>
 #include "page0zip.h"
+#include "backup_debug.h"
 
 char *tool_name;
-char tool_args[2048];
+char tool_args[8192];
 
 ulong mysql_server_version;
 
 /* server capabilities */
 bool have_changed_page_bitmaps = false;
-bool have_backup_locks = false;
 bool have_lock_wait_timeout = false;
 bool have_galera_enabled = false;
 bool have_multi_threaded_slave = false;
@@ -92,11 +98,54 @@ MYSQL *mysql_connection;
 
 extern my_bool opt_ssl_verify_server_cert, opt_use_ssl;
 
+
+/*
+  get_os_user()
+  Ressemles read_user_name() from libmariadb/libmariadb/mariadb_lib.c.
+*/
+
+#if !defined(_WIN32)
+
+#if defined(HAVE_GETPWUID) && defined(NO_GETPWUID_DECL)
+struct passwd *getpwuid(uid_t);
+char* getlogin(void);
+#endif
+
+static const char *get_os_user() // Posix
+{
+  if (!geteuid())
+    return "root";
+#ifdef HAVE_GETPWUID
+  struct passwd *pw;
+  const char *str;
+  if ((pw= getpwuid(geteuid())) != NULL)
+    return pw->pw_name;
+  if ((str= getlogin()) != NULL)
+    return str;
+#endif
+  if ((str= getenv("USER")) ||
+      (str= getenv("LOGNAME")) ||
+      (str= getenv("LOGIN")))
+    return str;
+  return NULL;
+}
+
+#else
+
+static const char *get_os_user() // Windows
+{
+  return getenv("USERNAME");
+}
+
+#endif // _WIN32
+
+
 MYSQL *
 xb_mysql_connect()
 {
 	MYSQL *connection = mysql_init(NULL);
 	char mysql_port_str[std::numeric_limits<int>::digits10 + 3];
+	const char *user= opt_user ? opt_user : get_os_user();
 
 	sprintf(mysql_port_str, "%d", opt_port);
 
@@ -126,7 +175,7 @@ xb_mysql_connect()
 
 	msg("Connecting to MariaDB server host: %s, user: %s, password: %s, "
 	       "port: %s, socket: %s", opt_host ? opt_host : "localhost",
-	       opt_user ? opt_user : "not set",
+	       user ? user : "not set",
 	       opt_password ? "set" : "not set",
 	       opt_port != 0 ? mysql_port_str : "not set",
 	       opt_socket ? opt_socket : "not set");
@@ -149,7 +198,7 @@ xb_mysql_connect()
 
 	if (!mysql_real_connect(connection,
 				opt_host ? opt_host : "localhost",
-				opt_user,
+				user,
 				opt_password,
 				"" /*database*/, opt_port,
 				opt_socket, 0)) {
@@ -205,13 +254,14 @@ struct mysql_variable {
 
 
 static
-void
+uint
 read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars,
 	bool vertical_result)
 {
 	MYSQL_RES *mysql_result;
 	MYSQL_ROW row;
 	mysql_variable *var;
+	uint n_values=0;
 
 	mysql_result = xb_mysql_query(connection, query, true);
 
@@ -225,6 +275,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars,
 				if (strcmp(var->name, name) == 0
 				    && value != NULL) {
 					*(var->value) = strdup(value);
+					n_values++;
 				}
 			}
 		}
@@ -241,6 +292,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars,
 					if (strcmp(var->name, name) == 0
 					    && value != NULL) {
 						*(var->value) = strdup(value);
+						n_values++;
 					}
 				}
 				++i;
@@ -249,6 +301,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars,
 	}
 
 	mysql_free_result(mysql_result);
+	return n_values;
 }
 
 
@@ -313,7 +366,6 @@ bool get_mysql_vars(MYSQL *connection)
 {
   char *gtid_mode_var= NULL;
   char *version_var= NULL;
-  char *have_backup_locks_var= NULL;
   char *log_bin_var= NULL;
   char *lock_wait_timeout_var= NULL;
   char *wsrep_on_var= NULL;
@@ -338,7 +390,6 @@ bool get_mysql_vars(MYSQL *connection)
   bool ret= true;
 
   mysql_variable mysql_vars[]= {
-      {"have_backup_locks", &have_backup_locks_var},
       {"log_bin", &log_bin_var},
       {"lock_wait_timeout", &lock_wait_timeout_var},
       {"gtid_mode", &gtid_mode_var},
@@ -363,11 +414,6 @@ bool get_mysql_vars(MYSQL *connection)
 
   read_mysql_variables(connection, "SHOW VARIABLES", mysql_vars, true);
 
-  if (have_backup_locks_var != NULL && !opt_no_backup_locks)
-  {
-    have_backup_locks= true;
-  }
-
   if (opt_binlog_info == BINLOG_INFO_AUTO)
   {
     if (log_bin_var != NULL && !strcmp(log_bin_var, "ON"))
@@ -514,24 +560,6 @@ Query the server to find out what backup capabilities it supports.
 bool
 detect_mysql_capabilities_for_backup()
 {
-	const char *query = "SELECT 'INNODB_CHANGED_PAGES', COUNT(*) FROM "
-				"INFORMATION_SCHEMA.PLUGINS "
-			    "WHERE PLUGIN_NAME LIKE 'INNODB_CHANGED_PAGES'";
-	char *innodb_changed_pages = NULL;
-	mysql_variable vars[] = {
-		{"INNODB_CHANGED_PAGES", &innodb_changed_pages}, {NULL, NULL}};
-
-	if (xtrabackup_incremental) {
-
-		read_mysql_variables(mysql_connection, query, vars, true);
-
-		ut_ad(innodb_changed_pages != NULL);
-
-		have_changed_page_bitmaps = (atoi(innodb_changed_pages) == 1);
-
-		free_mysql_variables(vars);
-	}
-
 	/* do some sanity checks */
 	if (opt_galera_info && !have_galera_enabled) {
 		msg("--galera-info is specified on the command "
@@ -839,11 +867,11 @@ static void stop_query_killer()
 
 
 /*********************************************************************//**
-Function acquires either a backup tables lock, if supported
+Function acquires backup locks
-by the server, or a global read lock (FLUSH TABLES WITH READ LOCK)
-otherwise.
 @returns true if lock acquired */
-bool lock_tables(MYSQL *connection)
+
+bool
+lock_for_backup_stage_start(MYSQL *connection)
 {
   if (have_lock_wait_timeout || opt_lock_wait_timeout)
   {
@@ -856,12 +884,6 @@ bool lock_tables(MYSQL *connection)
     xb_mysql_query(connection, buf, false);
   }
 
-  if (have_backup_locks)
-  {
-    msg("Executing LOCK TABLES FOR BACKUP...");
-    xb_mysql_query(connection, "LOCK TABLES FOR BACKUP", false);
-    return (true);
-  }
 
   if (opt_lock_wait_timeout)
   {
@@ -886,8 +908,6 @@ bool lock_tables(MYSQL *connection)
 
   xb_mysql_query(connection, "BACKUP STAGE START", true);
   DBUG_MARIABACKUP_EVENT("after_backup_stage_start", {});
-  xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true);
-  DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {});
   /* Set the maximum supported session value for
   lock_wait_timeout to prevent unnecessary timeouts when the
   global value is changed from the default */
@@ -903,24 +923,68 @@ bool lock_tables(MYSQL *connection)
   return (true);
 }
 
-/*********************************************************************//**
-If backup locks are used, execute LOCK BINLOG FOR BACKUP provided that we are
-not in the --no-lock mode and the lock has not been acquired already.
-@returns true if lock acquired */
 bool
-lock_binlog_maybe(MYSQL *connection)
+lock_for_backup_stage_flush(MYSQL *connection) {
-{
+	if (opt_kill_long_queries_timeout) {
-	if (have_backup_locks && !opt_no_lock && !binlog_locked) {
+		start_query_killer();
-		msg("Executing LOCK BINLOG FOR BACKUP...");
+	}
-		xb_mysql_query(connection, "LOCK BINLOG FOR BACKUP", false);
+	xb_mysql_query(connection, "BACKUP STAGE FLUSH", true);
-		binlog_locked = true;
+	if (opt_kill_long_queries_timeout) {
-
+		stop_query_killer();
-		return(true);
+	}
+	return true;
 }
 
-	return(false);
+bool
+lock_for_backup_stage_block_ddl(MYSQL *connection) {
+	if (opt_kill_long_queries_timeout) {
+		start_query_killer();
+	}
+	xb_mysql_query(connection, "BACKUP STAGE BLOCK_DDL", true);
+	DBUG_MARIABACKUP_EVENT("after_backup_stage_block_ddl", {});
+	if (opt_kill_long_queries_timeout) {
+		stop_query_killer();
+	}
+	return true;
 }
 
+bool
+lock_for_backup_stage_commit(MYSQL *connection) {
+	if (opt_kill_long_queries_timeout) {
+		start_query_killer();
+	}
+	xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true);
+	DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {});
+	if (opt_kill_long_queries_timeout) {
+		stop_query_killer();
+	}
+	return true;
+}
+
+bool backup_lock(MYSQL *con, const char *table_name) {
+	static const std::string backup_lock_prefix("BACKUP LOCK ");
+	std::string backup_lock_query = backup_lock_prefix + table_name;
+	xb_mysql_query(con, backup_lock_query.c_str(), true);
+	return true;
+}
+
+bool backup_unlock(MYSQL *con) {
+	xb_mysql_query(con, "BACKUP UNLOCK", true);
+	return true;
+}
+
+std::unordered_set<std::string>
+get_tables_in_use(MYSQL *con) {
+	std::unordered_set<std::string> result;
+	MYSQL_RES *q_res =
+		xb_mysql_query(con, "SHOW OPEN TABLES WHERE In_use = 1", true);
+	while (MYSQL_ROW row = mysql_fetch_row(q_res)) {
+		auto tk = table_key(row[0], row[1]);
+		msg("Table %s is in use", tk.c_str());
+		result.insert(std::move(tk));
+	}
+	return result;
+}
 
 /*********************************************************************//**
 Releases either global read lock acquired with FTWRL and the binlog
@@ -1355,15 +1419,33 @@ write_slave_info(ds_ctxt *datasink, MYSQL *connection)
 
 
 /*********************************************************************//**
-Retrieves MySQL Galera and
+Retrieves MySQL Galera and saves it in a file. It also prints it to stdout.
-saves it in a file. It also prints it to stdout. */
+
+We should create xtrabackup_galelera_info file even when backup locks
+are used because donor's wsrep_gtid_domain_id is needed later in joiner.
+Note that at this stage wsrep_local_state_uuid and wsrep_last_committed
+are inconsistent but they are not used in joiner. Joiner will rewrite this file
+at mariabackup --prepare phase and thus there is extra file donor_galera_info.
+Information is needed to maitain wsrep_gtid_domain_id and gtid_binlog_pos
+same across the cluster. If joiner node have different wsrep_gtid_domain_id
+we should still receive effective domain id from the donor node,
+and use it.
+*/
 bool
 write_galera_info(ds_ctxt *datasink, MYSQL *connection)
 {
   char *state_uuid = NULL, *state_uuid55 = NULL;
   char *last_committed = NULL, *last_committed55 = NULL;
   char *domain_id = NULL, *domain_id55 = NULL;
-	bool result;
+  bool result=true;
+  uint n_values=0;
+  char *wsrep_on = NULL, *wsrep_on55 = NULL;
+
+  mysql_variable vars[] = {
+    {"Wsrep_on", &wsrep_on},
+    {"wsrep_on", &wsrep_on55},
+    {NULL, NULL}
+  };
 
   mysql_variable status[] = {
     {"Wsrep_local_state_uuid", &state_uuid},
@@ -1379,27 +1461,30 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection)
     {NULL, NULL}
   };
 
-	/* When backup locks are supported by the server, we should skip
+  n_values= read_mysql_variables(connection, "SHOW VARIABLES", vars, true);
-	creating MB_GALERA_INFO file on the backup stage, because
+
-	wsrep_local_state_uuid and wsrep_last_committed will be inconsistent
+  if (n_values == 0 || (wsrep_on == NULL && wsrep_on55 == NULL))
-	without blocking commits. The state file will be created on the prepare
+  {
-	stage using the WSREP recovery procedure. */
+    msg("Server is not Galera node thus --galera-info does not "
-	if (have_backup_locks) {
+	"have any effect.");
-		return(true);
+    result = true;
+    goto cleanup;
   }
 
   read_mysql_variables(connection, "SHOW STATUS", status, true);
 
   if ((state_uuid == NULL && state_uuid55 == NULL)
-		|| (last_committed == NULL && last_committed55 == NULL)) {
+      || (last_committed == NULL && last_committed55 == NULL))
+  {
     msg("Warning: failed to get master wsrep state from SHOW STATUS.");
     result = true;
     goto cleanup;
   }
 
-	read_mysql_variables(connection, "SHOW VARIABLES LIKE 'wsrep%'", value, true);
+  n_values= read_mysql_variables(connection, "SHOW VARIABLES LIKE 'wsrep%'", value, true);
 
-	if (domain_id == NULL && domain_id55 == NULL) {
+  if (n_values == 0 || (domain_id == NULL && domain_id55 == NULL))
+  {
     msg("Warning: failed to get master wsrep state from SHOW VARIABLES.");
     result = true;
     goto cleanup;
@@ -1417,10 +1502,15 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection)
       last_committed ? last_committed : last_committed55,
       domain_id ? domain_id : domain_id55);
   }
+
   if (result)
-	{
     write_current_binlog_file(datasink, connection);
-	}
+
+  if (result)
+    msg("Writing Galera info succeeded with %s:%s %s",
+      state_uuid ? state_uuid : state_uuid55,
+      last_committed ? last_committed : last_committed55,
+      domain_id ? domain_id : domain_id55);
 
 cleanup:
   free_mysql_variables(status);
@@ -1468,8 +1558,6 @@ write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection)
 	if (gtid_exists) {
 		size_t log_bin_dir_length;
 
-		lock_binlog_maybe(connection);
-
 		xb_mysql_query(connection, "FLUSH BINARY LOGS", false);
 
 		read_mysql_variables(connection, "SHOW MASTER STATUS",
@@ -1828,13 +1916,13 @@ bool write_backup_config_file(ds_ctxt *datasink)
 		srv_log_file_size,
 		srv_page_size,
 		srv_undo_dir,
-		srv_undo_tablespaces,
+                (uint) srv_undo_tablespaces,
 		page_zip_level,
 		innobase_buffer_pool_filename ?
 			"innodb_buffer_pool_filename=" : "",
 		innobase_buffer_pool_filename ?
 			innobase_buffer_pool_filename : "",
-		xb_plugin_get_config());
+		encryption_plugin_get_config());
 		return rc;
 }
 
@@ -1853,9 +1941,11 @@ char *make_argv(char *buf, size_t len, int argc, char **argv)
 		if (strncmp(*argv, "--password", strlen("--password")) == 0) {
 			arg = "--password=...";
 		}
-		left-= snprintf(buf + len - left, left,
+		uint l= snprintf(buf + len - left, left,
 				"%s%c", arg, argc > 1 ? ' ' : 0);
 		++argv; --argc;
+                if (l < left)
+                  left-= l;
 	}
 
 	return buf;
@@ -1884,18 +1974,6 @@ select_history()
 	return(true);
 }
 
-bool
-flush_changed_page_bitmaps()
-{
-	if (xtrabackup_incremental && have_changed_page_bitmaps &&
-	    !xtrabackup_incremental_force_scan) {
-		xb_mysql_query(mysql_connection,
-			"FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS", false);
-	}
-	return(true);
-}
-
-
 /*********************************************************************//**
 Deallocate memory, disconnect from server, etc.
 @return	true on success. */
@@ -1971,3 +2049,23 @@ mdl_unlock_all()
   mysql_close(mdl_con);
   spaceid_to_tablename.clear();
 }
+
+ulonglong get_current_lsn(MYSQL *connection)
+{
+	static const char lsn_prefix[] = "\nLog sequence number ";
+	ulonglong lsn = 0;
+	if (MYSQL_RES *res = xb_mysql_query(connection,
+					    "SHOW ENGINE INNODB STATUS",
+					    true, false)) {
+		if (MYSQL_ROW row = mysql_fetch_row(res)) {
+			const char *p= strstr(row[2], lsn_prefix);
+			DBUG_ASSERT(p);
+			if (p) {
+				p += sizeof lsn_prefix - 1;
+				lsn = lsn_t(strtoll(p, NULL, 10));
+			}
+		}
+		mysql_free_result(res);
+	}
+	return lsn;
+}

extra/mariabackup/backup_mysql.h

@@ -2,13 +2,15 @@
 #define XTRABACKUP_BACKUP_MYSQL_H
 
 #include <mysql.h>
+#include <string>
+#include <unordered_set>
+#include "datasink.h"
 
 /* MariaDB version */
 extern ulong mysql_server_version;
 
 /* server capabilities */
 extern bool have_changed_page_bitmaps;
-extern bool have_backup_locks;
 extern bool have_lock_wait_timeout;
 extern bool have_galera_enabled;
 extern bool have_multi_threaded_slave;
@@ -35,9 +37,6 @@ capture_tool_command(int argc, char **argv);
 bool
 select_history();
 
-bool
-flush_changed_page_bitmaps();
-
 void
 backup_cleanup();
 
@@ -75,7 +74,21 @@ bool
 lock_binlog_maybe(MYSQL *connection);
 
 bool
-lock_tables(MYSQL *connection);
+lock_for_backup_stage_start(MYSQL *connection);
+
+bool
+lock_for_backup_stage_flush(MYSQL *connection);
+
+bool
+lock_for_backup_stage_block_ddl(MYSQL *connection);
+
+bool
+lock_for_backup_stage_commit(MYSQL *connection);
+
+bool backup_lock(MYSQL *con, const char *table_name);
+bool backup_unlock(MYSQL *con);
+
+std::unordered_set<std::string> get_tables_in_use(MYSQL *con);
 
 bool
 wait_for_safe_slave(MYSQL *connection);
@@ -86,5 +99,6 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection);
 bool
 write_slave_info(ds_ctxt *datasink, MYSQL *connection);
 
+ulonglong get_current_lsn(MYSQL *connection);
 
 #endif

extra/mariabackup/changed_page_bitmap.h

@@ -1,85 +0,0 @@
-/******************************************************
-XtraBackup: hot backup tool for InnoDB
-(c) 2009-2012 Percona Inc.
-Originally Created 3/3/2009 Yasufumi Kinoshita
-Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
-Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
-
-*******************************************************/
-
-/* Changed page bitmap interface */
-
-#ifndef XB_CHANGED_PAGE_BITMAP_H
-#define XB_CHANGED_PAGE_BITMAP_H
-
-#include <ut0rbt.h>
-#include <fil0fil.h>
-
-/* The changed page bitmap structure */
-typedef ib_rbt_t xb_page_bitmap;
-
-struct xb_page_bitmap_range_struct;
-
-/* The bitmap range iterator over one space id */
-typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range;
-
-/****************************************************************//**
-Read the disk bitmap and build the changed page bitmap tree for the
-LSN interval incremental_lsn to log_sys.next_checkpoint_lsn.
-
-@return the built bitmap tree */
-xb_page_bitmap*
-xb_page_bitmap_init(void);
-/*=====================*/
-
-/****************************************************************//**
-Free the bitmap tree. */
-void
-xb_page_bitmap_deinit(
-/*==================*/
-	xb_page_bitmap*	bitmap);	/*!<in/out: bitmap tree */
-
-
-/****************************************************************//**
-Set up a new bitmap range iterator over a given space id changed
-pages in a given bitmap.
-
-@return bitmap range iterator */
-xb_page_bitmap_range*
-xb_page_bitmap_range_init(
-/*======================*/
-	xb_page_bitmap*	bitmap,		/*!< in: bitmap to iterate over */
-	ulint		space_id);	/*!< in: space id */
-
-/****************************************************************//**
-Get the next page id that has its bit set or cleared, i.e. equal to
-bit_value.
-
-@return page id */
-ulint
-xb_page_bitmap_range_get_next_bit(
-/*==============================*/
-	xb_page_bitmap_range*	bitmap_range,	/*!< in/out: bitmap range */
-	ibool			bit_value);	/*!< in: bit value */
-
-/****************************************************************//**
-Free the bitmap range iterator. */
-void
-xb_page_bitmap_range_deinit(
-/*========================*/
-	xb_page_bitmap_range*	bitmap_range);	/*! in/out: bitmap range */
-
-#endif

extra/mariabackup/common.h

@@ -23,7 +23,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 
 #include <my_global.h>
 #include <mysql_version.h>
-#include <fcntl.h>
 #include <stdarg.h>
 #include <my_sys.h>
 
@@ -143,7 +142,7 @@ static inline ATTRIBUTE_FORMAT(printf, 1,2) ATTRIBUTE_NORETURN void die(const ch
 # define POSIX_FADV_NORMAL
 # define POSIX_FADV_SEQUENTIAL
 # define POSIX_FADV_DONTNEED
-# define posix_fadvise(a,b,c,d) do {} while(0)
+# define posix_fadvise(fd, offset, len, advice) do { (void)offset; } while(0)
 #endif
 
 /***********************************************************************

extra/mariabackup/common_engine.cc

@@ -0,0 +1,512 @@
+#include "common_engine.h"
+#include "backup_copy.h"
+#include "xtrabackup.h"
+#include "common.h"
+#include "backup_debug.h"
+
+#include <unordered_map>
+#include <atomic>
+#include <memory>
+#include <chrono>
+
+namespace common_engine {
+
+class Table {
+public:
+	Table(std::string &db, std::string &table, std::string &fs_name) :
+		m_db(std::move(db)), m_table(std::move(table)),
+		m_fs_name(std::move(fs_name)) {}
+	virtual ~Table() {}
+	void add_file_name(const char *file_name) { m_fnames.push_back(file_name); }
+	virtual bool copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock,
+		bool finalize, unsigned thread_num);
+	std::string &get_db() { return m_db; }
+	std::string &get_table() { return m_table; }
+	std::string &get_version() { return m_version; }
+
+protected:
+	std::string m_db;
+	std::string m_table;
+	std::string m_fs_name;
+	std::string m_version;
+	std::vector<std::string> m_fnames;
+};
+
+bool
+Table::copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool, unsigned thread_num) {
+	static const size_t buf_size = 10 * 1024 * 1024;
+	std::unique_ptr<uchar[]> buf;
+	bool result = false;
+	File frm_file = -1;
+	std::vector<File> files;
+	bool locked = false;
+	std::string full_tname("`");
+	full_tname.append(m_db).append("`.`").append(m_table).append("`");
+
+	if (!no_lock && !backup_lock(con, full_tname.c_str())) {
+		msg(thread_num, "Error on executing BACKUP LOCK for table %s",
+			full_tname.c_str());
+		goto exit;
+	}
+	else
+		locked = !no_lock;
+
+	if ((frm_file = mysql_file_open(key_file_frm, (m_fs_name + ".frm").c_str(),
+		O_RDONLY | O_SHARE, MYF(0))) < 0 && !m_fnames.empty() &&
+		!ends_with(m_fnames[0].c_str(), ".ARZ") &&
+		!ends_with(m_fnames[0].c_str(), ".ARM")) {
+		// Don't treat it as error, as the table can be dropped after it
+		// was added to queue for copying
+		result = true;
+		goto exit;
+	}
+
+	for (const auto &fname : m_fnames) {
+		File file = mysql_file_open(0, fname.c_str(),O_RDONLY | O_SHARE, MYF(0));
+		if (file < 0) {
+			msg(thread_num, "Error on file %s open during %s table copy",
+				fname.c_str(), full_tname.c_str());
+			goto exit;
+		}
+		files.push_back(file);
+	}
+
+	if (locked && !backup_unlock(con)) {
+		msg(thread_num, "Error on BACKUP UNLOCK for table %s", full_tname.c_str());
+		locked = false;
+		goto exit;
+	}
+
+	locked = false;
+
+	buf.reset(new uchar[buf_size]);
+
+	for (size_t i = 0; i < m_fnames.size(); ++i) {
+		ds_file_t *dst_file = nullptr;
+		size_t bytes_read;
+		size_t copied_size = 0;
+		MY_STAT stat_info;
+
+		if (my_fstat(files[i], &stat_info, MYF(0))) {
+			msg(thread_num, "error: failed to get stat info for file %s of "
+					"table %s", m_fnames[i].c_str(), full_tname.c_str());
+			goto exit;
+		}
+
+		const char	*dst_path =
+			(xtrabackup_copy_back || xtrabackup_move_back) ?
+			m_fnames[i].c_str() : trim_dotslash(m_fnames[i].c_str());
+
+		dst_file = ds_open(ds, dst_path, &stat_info, false);
+		if (!dst_file) {
+			msg(thread_num, "error: cannot open destination stream for %s, table %s",
+				dst_path, full_tname.c_str());
+			goto exit;
+		}
+
+		while ((bytes_read = my_read(files[i], buf.get(), buf_size, MY_WME))) {
+			if (bytes_read == size_t(-1)) {
+				msg(thread_num, "error: file %s read for table %s",
+					m_fnames[i].c_str(), full_tname.c_str());
+				ds_close(dst_file);
+				goto exit;
+			}
+			xtrabackup_io_throttling();
+			if (ds_write(dst_file, buf.get(), bytes_read)) {
+				msg(thread_num, "error: file %s write for table %s",
+					dst_path, full_tname.c_str());
+				ds_close(dst_file);
+				goto exit;
+			}
+			copied_size += bytes_read;
+		}
+		mysql_file_close(files[i], MYF(MY_WME));
+		files[i] = -1;
+		ds_close(dst_file);
+		msg(thread_num, "Copied file %s for table %s, %zu bytes",
+			m_fnames[i].c_str(), full_tname.c_str(), copied_size);
+	}
+
+	result = true;
+
+#ifndef DBUG_OFF
+	{
+		std::string sql_name(m_db);
+		sql_name.append("/").append(m_table);
+		DBUG_MARIABACKUP_EVENT_LOCK("after_ce_table_copy", fil_space_t::name_type(sql_name.data(), sql_name.size()));
+	}
+#endif // DBUG_OFF
+exit:
+	if (frm_file >= 0) {
+		m_version = ::read_table_version_id(frm_file);
+		mysql_file_close(frm_file, MYF(MY_WME));
+	}
+	if (locked && !backup_unlock(con)) {
+		msg(thread_num, "Error on BACKUP UNLOCK for table %s", full_tname.c_str());
+		result = false;
+	}
+	for (auto file : files)
+		if (file >= 0)
+			mysql_file_close(file, MYF(MY_WME));
+	return result;
+}
+
+// Append-only tables
+class LogTable : public Table {
+	public:
+		LogTable(std::string &db, std::string &table, std::string &fs_name) :
+			Table(db, table, fs_name) {}
+
+		virtual ~LogTable() { (void)close(); }
+		bool
+			copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool finalize,
+			unsigned thread_num) override;
+		bool close();
+	private:
+		bool open(ds_ctxt_t *ds, unsigned thread_num);
+		std::vector<File> m_src;
+		std::vector<ds_file_t *> m_dst;
+};
+
+bool
+LogTable::open(ds_ctxt_t *ds, unsigned thread_num) {
+	DBUG_ASSERT(m_src.empty());
+	DBUG_ASSERT(m_dst.empty());
+
+	std::string full_tname("`");
+	full_tname.append(m_db).append("`.`").append(m_table).append("`");
+
+	for (const auto &fname : m_fnames) {
+		File file = mysql_file_open(0, fname.c_str(),O_RDONLY | O_SHARE, MYF(0));
+		if (file < 0) {
+			msg(thread_num, "Error on file %s open during %s log table copy",
+				fname.c_str(), full_tname.c_str());
+			return false;
+		}
+		m_src.push_back(file);
+
+		MY_STAT stat_info;
+		if (my_fstat(file, &stat_info, MYF(0))) {
+			msg(thread_num, "error: failed to get stat info for file %s of "
+					"log table %s", fname.c_str(), full_tname.c_str());
+			return false;
+		}
+		const char	*dst_path =
+			(xtrabackup_copy_back || xtrabackup_move_back) ?
+			fname.c_str() : trim_dotslash(fname.c_str());
+		ds_file_t *dst_file = ds_open(ds, dst_path, &stat_info, false);
+		if (!dst_file) {
+			msg(thread_num, "error: cannot open destination stream for %s, "
+				"log table %s", dst_path, full_tname.c_str());
+			return false;
+		}
+		m_dst.push_back(dst_file);
+	}
+
+	File frm_file;
+	if ((frm_file = mysql_file_open(key_file_frm, (m_fs_name + ".frm").c_str(),
+		O_RDONLY | O_SHARE, MYF(0))) < 0 && !m_fnames.empty() &&
+		!ends_with(m_fnames[0].c_str(), ".ARZ") &&
+		!ends_with(m_fnames[0].c_str(), ".ARM")) {
+		msg(thread_num, "Error on .frm file open for log table %s",
+			full_tname.c_str());
+		return false;
+	}
+
+	m_version = ::read_table_version_id(frm_file);
+	mysql_file_close(frm_file, MYF(MY_WME));
+
+	return true;
+}
+
+bool LogTable::close() {
+	while (!m_src.empty()) {
+		auto f = m_src.back();
+		m_src.pop_back();
+		mysql_file_close(f, MYF(MY_WME));
+	}
+	while (!m_dst.empty()) {
+		auto f = m_dst.back();
+		m_dst.pop_back();
+		ds_close(f);
+	}
+	return true;
+}
+
+bool
+LogTable::copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool finalize,
+	unsigned thread_num) {
+	static const size_t buf_size = 10 * 1024 * 1024;
+	DBUG_ASSERT(ds);
+	DBUG_ASSERT(con);
+	if (m_src.empty() && !open(ds, thread_num)) {
+		close();
+		return false;
+	}
+	DBUG_ASSERT(m_src.size() == m_dst.size());
+
+	std::unique_ptr<uchar[]> buf(new uchar[buf_size]);
+	for (size_t i = 0; i < m_src.size(); ++i) {
+		// .CSM can be rewritten (see write_meta_file() usage in ha_tina.cc)
+		if (!finalize && ends_with(m_fnames[i].c_str(), ".CSM"))
+			continue;
+		size_t bytes_read;
+		size_t copied_size = 0;
+		while ((bytes_read = my_read(m_src[i], buf.get(), buf_size, MY_WME))) {
+			if (bytes_read == size_t(-1)) {
+				msg(thread_num, "error: file %s read for log table %s",
+					m_fnames[i].c_str(),
+					std::string("`").append(m_db).append("`.`").
+					append(m_table).append("`").c_str());
+				close();
+				return false;
+			}
+			xtrabackup_io_throttling();
+			if (ds_write(m_dst[i], buf.get(), bytes_read)) {
+				msg(thread_num, "error: file %s write for log table %s",
+					m_fnames[i].c_str(), std::string("`").append(m_db).append("`.`").
+					append(m_table).append("`").c_str());
+				close();
+				return false;
+			}
+			copied_size += bytes_read;
+		}
+		msg(thread_num, "Copied file %s for log table %s, %zu bytes",
+			m_fnames[i].c_str(), std::string("`").append(m_db).append("`.`").
+					append(m_table).append("`").c_str(), copied_size);
+	}
+
+	return true;
+}
+
+class BackupImpl {
+	public:
+		BackupImpl(
+				const char *datadir_path, ds_ctxt_t *datasink,
+				std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool) :
+			m_datadir_path(datadir_path), m_ds(datasink), m_con_pool(con_pool),
+			m_process_table_jobs(thread_pool) {}
+		~BackupImpl() { }
+		bool scan(
+				const std::unordered_set<std::string> &exclude_tables,
+				std::unordered_set<std::string> *out_processed_tables,
+				bool no_lock, bool collect_log_and_stats);
+		void set_post_copy_table_hook(const post_copy_table_hook_t &hook) {
+			m_table_post_copy_hook = hook;
+		}
+		bool copy_log_tables(bool finalize);
+		bool copy_stats_tables();
+		bool wait_for_finish();
+		bool close_log_tables();
+	private:
+
+		void process_table_job(Table *table, bool no_lock, bool delete_table,
+			bool finalize, unsigned thread_num);
+
+		const char *m_datadir_path;
+		ds_ctxt_t *m_ds;
+		std::vector<MYSQL *> &m_con_pool;
+		TasksGroup m_process_table_jobs;
+
+		post_copy_table_hook_t m_table_post_copy_hook;
+		std::unordered_map<table_key_t, std::unique_ptr<LogTable>> m_log_tables;
+		std::unordered_map<table_key_t, std::unique_ptr<Table>> m_stats_tables;
+};
+
+void BackupImpl::process_table_job(Table *table, bool no_lock,
+	bool delete_table, bool finalize, unsigned thread_num) {
+	int result = 0;
+
+	if (!m_process_table_jobs.get_result())
+		goto exit;
+
+	if (!table->copy(m_ds, m_con_pool[thread_num], no_lock, finalize, thread_num))
+		goto exit;
+
+	if (m_table_post_copy_hook)
+		m_table_post_copy_hook(table->get_db(), table->get_table(),
+		table->get_version());
+
+	result = 1;
+
+exit:
+	if (delete_table)
+		delete table;
+	m_process_table_jobs.finish_task(result);
+}
+
+bool BackupImpl::scan(const std::unordered_set<table_key_t> &exclude_tables,
+	std::unordered_set<table_key_t> *out_processed_tables, bool no_lock,
+	bool collect_log_and_stats) {
+
+	msg("Start scanning common engine tables, need backup locks: %d, "
+		"collect log and stat tables: %d", no_lock, collect_log_and_stats);
+
+	std::unordered_map<table_key_t, std::unique_ptr<Table>> found_tables;
+
+	foreach_file_in_db_dirs(m_datadir_path,
+		[&](const char *file_path)->bool {
+
+		static const char *ext_list[] =
+			{".MYD", ".MYI", ".MRG", ".ARM", ".ARZ", ".CSM", ".CSV", NULL};
+
+		bool is_aria = ends_with(file_path, ".MAD") || ends_with(file_path, ".MAI");
+
+		if (!collect_log_and_stats && is_aria)
+			return true;
+
+		if (!is_aria && !filename_matches(file_path, ext_list))
+			return true;
+
+		if (check_if_skip_table(file_path)) {
+			msg("Skipping %s.", file_path);
+			return true;
+		}
+
+		auto db_table_fs = convert_filepath_to_tablename(file_path);
+		auto tk =
+			table_key(std::get<0>(db_table_fs), std::get<1>(db_table_fs));
+
+		// log and stats tables are only collected in this function,
+		// so there is no need to filter out them with exclude_tables.
+		if (collect_log_and_stats) {
+			if (is_log_table(std::get<0>(db_table_fs).c_str(),
+				std::get<1>(db_table_fs).c_str())) {
+				auto table_it = m_log_tables.find(tk);
+				if (table_it == m_log_tables.end()) {
+					msg("Log table found: %s", tk.c_str());
+					table_it = m_log_tables.emplace(tk,
+						std::unique_ptr<LogTable>(new LogTable(std::get<0>(db_table_fs),
+						std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first;
+				}
+				msg("Collect log table file: %s", file_path);
+				table_it->second->add_file_name(file_path);
+				return true;
+			}
+			// Aria can handle statistics tables
+			else if (is_stats_table(std::get<0>(db_table_fs).c_str(),
+				std::get<1>(db_table_fs).c_str()) && !is_aria) {
+				auto table_it = m_stats_tables.find(tk);
+				if (table_it == m_stats_tables.end()) {
+					msg("Stats table found: %s", tk.c_str());
+					table_it = m_stats_tables.emplace(tk,
+						std::unique_ptr<Table>(new Table(std::get<0>(db_table_fs),
+						std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first;
+				}
+				msg("Collect stats table file: %s", file_path);
+				table_it->second->add_file_name(file_path);
+				return true;
+			}
+		} else if (is_log_table(std::get<0>(db_table_fs).c_str(),
+			std::get<1>(db_table_fs).c_str()) ||
+			is_stats_table(std::get<0>(db_table_fs).c_str(),
+			std::get<1>(db_table_fs).c_str()))
+			return true;
+
+		if (is_aria)
+			return true;
+
+		if (exclude_tables.count(tk)) {
+			msg("Skip table %s at it is in exclude list", tk.c_str());
+			return true;
+		}
+
+		auto table_it = found_tables.find(tk);
+		if (table_it == found_tables.end()) {
+			table_it = found_tables.emplace(tk,
+				std::unique_ptr<Table>(new Table(std::get<0>(db_table_fs),
+				std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first;
+		}
+
+		table_it->second->add_file_name(file_path);
+
+		return true;
+	});
+
+	for (auto &table_it : found_tables) {
+		m_process_table_jobs.push_task(
+			std::bind(&BackupImpl::process_table_job, this, table_it.second.release(),
+					no_lock, true, false, std::placeholders::_1));
+		if (out_processed_tables)
+			out_processed_tables->insert(table_it.first);
+	}
+
+	msg("Stop scanning common engine tables");
+	return true;
+}
+
+bool BackupImpl::copy_log_tables(bool finalize) {
+	for (auto &table_it : m_log_tables) {
+		// Do not execute BACKUP LOCK for log tables as it's supposed
+		// that they must be copied on BLOCK_DDL and BLOCK_COMMIT locks.
+		m_process_table_jobs.push_task(
+			std::bind(&BackupImpl::process_table_job, this, table_it.second.get(),
+					true, false, finalize, std::placeholders::_1));
+	}
+	return true;
+}
+
+bool BackupImpl::copy_stats_tables() {
+	for (auto &table_it : m_stats_tables) {
+		// Do not execute BACKUP LOCK for stats tables as it's supposed
+		// that they must be copied on BLOCK_DDL and BLOCK_COMMIT locks.
+		// Delete stats table object after copy (see process_table_job())
+		m_process_table_jobs.push_task(
+			std::bind(&BackupImpl::process_table_job, this, table_it.second.release(),
+					true, true, false, std::placeholders::_1));
+	}
+	m_stats_tables.clear();
+	return true;
+}
+
+bool BackupImpl::wait_for_finish() {
+	/* Wait for threads to exit */
+	return m_process_table_jobs.wait_for_finish();
+}
+
+bool BackupImpl::close_log_tables() {
+	bool result = wait_for_finish();
+	for (auto &table_it : m_log_tables)
+		table_it.second->close();
+	return result;
+}
+
+Backup::Backup(const char *datadir_path, ds_ctxt_t *datasink,
+	std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool) :
+		m_backup_impl(
+			new BackupImpl(datadir_path, datasink, con_pool,
+			thread_pool)) { }
+
+Backup::~Backup() {
+	delete m_backup_impl;
+}
+
+bool Backup::scan(
+	const std::unordered_set<table_key_t> &exclude_tables,
+	std::unordered_set<table_key_t> *out_processed_tables,
+	bool no_lock, bool collect_log_and_stats) {
+	return m_backup_impl->scan(exclude_tables, out_processed_tables, no_lock,
+		collect_log_and_stats);
+}
+
+bool Backup::copy_log_tables(bool finalize) {
+	return m_backup_impl->copy_log_tables(finalize);
+}
+
+bool Backup::copy_stats_tables() {
+	return m_backup_impl->copy_stats_tables();
+}
+
+bool Backup::wait_for_finish() {
+	return m_backup_impl->wait_for_finish();
+}
+
+bool Backup::close_log_tables() {
+	return m_backup_impl->close_log_tables();
+}
+
+void Backup::set_post_copy_table_hook(const post_copy_table_hook_t &hook) {
+	m_backup_impl->set_post_copy_table_hook(hook);
+}
+
+} // namespace common_engine

extra/mariabackup/common_engine.h

@@ -0,0 +1,39 @@
+#pragma once
+#include "my_global.h"
+#include "backup_mysql.h"
+#include "datasink.h"
+#include "thread_pool.h"
+#include "xtrabackup.h"
+
+#include <unordered_set>
+#include <string>
+#include <vector>
+
+namespace common_engine {
+
+class BackupImpl;
+
+class Backup {
+	public:
+		Backup(const char *datadir_path, ds_ctxt_t *datasink,
+				std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool);
+		~Backup();
+		Backup (Backup &&other) = delete;
+		Backup & operator= (Backup &&other) = delete;
+		Backup(const Backup &) = delete;
+		Backup & operator= (const Backup &) = delete;
+		bool scan(
+				const std::unordered_set<table_key_t> &exclude_tables,
+				std::unordered_set<table_key_t> *out_processed_tables,
+				bool no_lock, bool collect_log_and_stats);
+		bool copy_log_tables(bool finalize);
+		bool copy_stats_tables();
+		bool wait_for_finish();
+		bool close_log_tables();
+		void set_post_copy_table_hook(const post_copy_table_hook_t &hook);
+	private:
+		BackupImpl *m_backup_impl;
+};
+
+} // namespace common_engine
+

extra/mariabackup/datasink.cc

@@ -80,11 +80,11 @@ ds_create(const char *root, ds_type_t type)
 /************************************************************************
 Open a datasink file */
 ds_file_t *
-ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat)
+ds_open(ds_ctxt_t *ctxt, const char *path, const MY_STAT *stat, bool rewrite)
 {
 	ds_file_t	*file;
 
-	file = ctxt->datasink->open(ctxt, path, stat);
+	file = ctxt->datasink->open(ctxt, path, stat, rewrite);
 	if (file != NULL) {
 		file->datasink = ctxt->datasink;
 	}
@@ -104,6 +104,30 @@ ds_write(ds_file_t *file, const void *buf, size_t len)
 	return file->datasink->write(file, (const uchar *)buf, len);
 }
 
+int ds_seek_set(ds_file_t *file, my_off_t offset) {
+	DBUG_ASSERT(file);
+	DBUG_ASSERT(file->datasink);
+	if (file->datasink->seek_set)
+		return file->datasink->seek_set(file, offset);
+	return 0;
+}
+
+int ds_rename(ds_ctxt_t *ctxt, const char *old_path, const char *new_path) {
+	DBUG_ASSERT(ctxt);
+	DBUG_ASSERT(ctxt->datasink);
+	if (ctxt->datasink->rename)
+		return ctxt->datasink->rename(ctxt, old_path, new_path);
+	return 0;
+}
+
+int ds_remove(ds_ctxt_t *ctxt, const char *path) {
+	DBUG_ASSERT(ctxt);
+	DBUG_ASSERT(ctxt->datasink);
+	if (ctxt->datasink->remove)
+		return ctxt->datasink->mremove(ctxt, path);
+	return 0;
+}
+
 /************************************************************************
 Close a datasink file.
 @return 0 on success, 1, on error. */

extra/mariabackup/datasink.h

@@ -43,7 +43,8 @@ typedef struct ds_ctxt {
         */
         bool copy_file(const char *src_file_path,
                        const char *dst_file_path,
-                       uint thread_n);
+                       uint thread_n,
+                       bool rewrite = false);
 
         bool move_file(const char *src_file_path,
                        const char *dst_file_path,
@@ -76,10 +77,15 @@ typedef struct {
 
 struct datasink_struct {
 	ds_ctxt_t *(*init)(const char *root);
-	ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
+	ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path,
+		const MY_STAT *stat, bool rewrite);
 	int (*write)(ds_file_t *file, const unsigned char *buf, size_t len);
+	int (*seek_set)(ds_file_t *file, my_off_t offset);
 	int (*close)(ds_file_t *file);
 	int (*remove)(const char *path);
+	// TODO: consider to return bool from "rename" and "remove"
+	int (*rename)(ds_ctxt_t *ctxt, const char *old_path, const char *new_path);
+	int (*mremove)(ds_ctxt_t *ctxt, const char *path);
 	void (*deinit)(ds_ctxt_t *ctxt);
 };
 
@@ -106,12 +112,17 @@ ds_ctxt_t *ds_create(const char *root, ds_type_t type);
 
 /************************************************************************
 Open a datasink file */
-ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
+ds_file_t *ds_open(
+	ds_ctxt_t *ctxt, const char *path, const MY_STAT *stat, bool rewrite = false);
 
 /************************************************************************
 Write to a datasink file.
 @return 0 on success, 1 on error. */
 int ds_write(ds_file_t *file, const void *buf, size_t len);
+int ds_seek_set(ds_file_t *file, my_off_t offset);
+
+int ds_rename(ds_ctxt_t *ctxt, const char *old_path, const char *new_path);
+int ds_remove(ds_ctxt_t *ctxt, const char *path);
 
 /************************************************************************
 Close a datasink file.

extra/mariabackup/ddl_log.cc

@@ -0,0 +1,553 @@
+#include "ddl_log.h"
+#include "common.h"
+#include "my_sys.h"
+#include "sql_table.h"
+#include "backup_copy.h"
+#include "xtrabackup.h"
+#include <unordered_set>
+#include <functional>
+#include <memory>
+#include <cstddef>
+
+namespace ddl_log {
+
+struct Entry {
+	enum Type {
+		CREATE,
+		ALTER,
+		RENAME,
+		REPAIR,
+		OPTIMIZE,
+		DROP,
+		TRUNCATE,
+		CHANGE_INDEX,
+		BULK_INSERT
+	};
+	Type type;
+	std::string date;
+	std::string engine;
+	bool partitioned;
+	std::string db;
+	std::string table;
+	std::string id;
+	std::string new_engine;
+	bool new_partitioned;
+	std::string new_db;
+	std::string new_table;
+	std::string new_id;
+};
+
+typedef std::vector<std::unique_ptr<Entry>> entries_t;
+typedef std::function<bool(std::unique_ptr<Entry>)> store_entry_func_t;
+
+const char *aria_engine_name = "Aria";
+static const char *frm_ext = ".frm";
+static const char *database_keyword = "DATABASE";
+
+const std::unordered_map<std::string, std::vector<const char *>> engine_exts =
+{
+	{"Aria", {".MAD", ".MAI"}},
+	{"MyISAM", {".MYD", ".MYI"}},
+	{"MRG_MyISAM", {".MRG"}},
+	{"ARCHIVE", {".ARM", ".ARZ"}},
+	{"CSV", {".CSM", ".CSV"}}
+};
+
+static inline bool known_engine(const std::string &engine) {
+	return engine_exts.count(engine);
+}
+
+// TODO: add error messages
+size_t parse(const uchar *buf, size_t buf_size, bool &error_flag,
+	store_entry_func_t &store_entry_func) {
+	DBUG_ASSERT(buf);
+	static constexpr char token_delimiter = '\t';
+	static constexpr char line_delimiter = '\n';
+	enum {
+		TOKEN_FIRST = 0,
+		TOKEN_DATE = TOKEN_FIRST,
+		TOKEN_TYPE,
+		TOKEN_ENGINE,
+		TOKEN_PARTITIONED,
+		TOKEN_DB,
+		TOKEN_TABLE,
+		TOKEN_ID,
+		TOKEN_MANDATORY = TOKEN_ID,
+		TOKEN_NEW_ENGINE,
+		TOKEN_NEW_PARTITIONED,
+		TOKEN_NEW_DB,
+		TOKEN_NEW_TABLE,
+		TOKEN_NEW_ID,
+		TOKEN_LAST = TOKEN_NEW_ID
+	};
+	const size_t string_offsets[TOKEN_LAST + 1] = {
+		offsetof(Entry, date),
+		offsetof(Entry, type), // not a string, be careful
+		offsetof(Entry, engine),
+		offsetof(Entry, partitioned), // not a string, be careful
+		offsetof(Entry, db),
+		offsetof(Entry, table),
+		offsetof(Entry, id),
+		offsetof(Entry, new_engine),
+		offsetof(Entry, new_partitioned), // not a string, be careful
+		offsetof(Entry, new_db),
+		offsetof(Entry, new_table),
+		offsetof(Entry, new_id)
+	};
+	const std::unordered_map<std::string, Entry::Type> str_to_type = {
+		{"CREATE", Entry::CREATE},
+		{"ALTER", Entry::ALTER},
+		{"RENAME", Entry::RENAME},
+		// TODO: fix to use uppercase-only
+		{"repair", Entry::REPAIR},
+		{"optimize", Entry::OPTIMIZE},
+		{"DROP", Entry::DROP},
+		{"TRUNCATE", Entry::TRUNCATE},
+		{"CHANGE_INDEX", Entry::CHANGE_INDEX},
+		{"BULK_INSERT", Entry::BULK_INSERT}
+	};
+
+	const uchar *new_line = buf;
+	const uchar *token_start = buf;
+	unsigned token_num = TOKEN_FIRST;
+
+	error_flag = false;
+
+	std::unique_ptr<Entry> entry(new Entry());
+
+	for (const uchar *ptr = buf; ptr < buf + buf_size; ++ptr) {
+
+		if (*ptr != token_delimiter && *ptr != line_delimiter)
+			continue;
+
+		if (token_start != ptr) {
+			std::string token(token_start, ptr);
+
+			if (token_num == TOKEN_TYPE) {
+				const auto type_it = str_to_type.find(token);
+				if (type_it == str_to_type.end()) {
+					error_flag = true;
+					goto exit;
+				}
+				entry->type = type_it->second;
+			}
+			else if (token_num == TOKEN_PARTITIONED) {
+				entry->partitioned = token[0] - '0';
+			}
+			else if (token_num == TOKEN_NEW_PARTITIONED) {
+				entry->new_partitioned = token[0] - '0';
+			}
+			else if (token_num <= TOKEN_LAST) {
+				DBUG_ASSERT(token_num != TOKEN_TYPE);
+				DBUG_ASSERT(token_num != TOKEN_PARTITIONED);
+				DBUG_ASSERT(token_num != TOKEN_NEW_PARTITIONED);
+				reinterpret_cast<std::string *>
+					(reinterpret_cast<uchar *>(entry.get()) + string_offsets[token_num])->
+					assign(std::move(token));
+			}
+			else {
+				error_flag = true;
+				goto exit;
+			}
+		}
+		token_start = ptr + 1;
+
+		if (*ptr == line_delimiter) {
+			if (token_num < TOKEN_MANDATORY) {
+				error_flag = true;
+				goto exit;
+			}
+			if (!store_entry_func(std::move(entry))) {
+				error_flag = true;
+				goto exit;
+			}
+			entry.reset(new Entry());
+			token_num = TOKEN_FIRST;
+			new_line = ptr + 1;
+		} else
+			++token_num;
+	}
+
+exit:
+	return new_line - buf;
+}
+
+bool parse(const char *file_path, store_entry_func_t store_entry_func) {
+	DBUG_ASSERT(file_path);
+	DBUG_ASSERT(store_entry_func);
+	File file= -1;
+	bool result = true;
+	uchar buf[1024];
+	size_t bytes_read = 0;
+	size_t buf_read_offset = 0;
+
+	if ((file= my_open(file_path, O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC,
+		MYF(MY_WME))) < 0) {
+		msg("DDL log file %s open failed: %d", file_path, my_errno);
+		result = false;
+		goto exit;
+	}
+
+	while((bytes_read = my_read(
+		file, &buf[buf_read_offset], sizeof(buf) - buf_read_offset, MY_WME)) > 0) {
+		if (bytes_read == size_t(-1)) {
+			msg("DDL log file %s read error: %d", file_path, my_errno);
+			result = false;
+			break;
+		}
+		bytes_read += buf_read_offset;
+		bool parse_error_flag = false;
+		size_t bytes_parsed = parse(
+			buf, bytes_read, parse_error_flag, store_entry_func);
+		if (parse_error_flag) {
+			result = false;
+			break;
+		}
+		size_t rest_size = bytes_read - bytes_parsed;
+		if (rest_size)
+			memcpy(buf, buf + bytes_parsed, rest_size);
+		buf_read_offset = rest_size;
+	}
+
+exit:
+	if (file >= 0)
+		my_close(file, MYF(MY_WME));
+	return result;
+};
+
+
+static
+bool process_database(
+	const char *datadir_path,
+	ds_ctxt_t *ds,
+	const Entry &entry,
+	std::unordered_set<std::string> &dropped_databases) {
+
+	if (entry.type == Entry::Type::CREATE ||
+		entry.type == Entry::Type::ALTER) {
+		std::string opt_file(datadir_path);
+		opt_file.append("/").append(entry.db).append("/db.opt");
+		if (!ds->copy_file(opt_file.c_str(), opt_file.c_str(), 0, true)) {
+			msg("Failed to re-copy %s.", opt_file.c_str());
+			return false;
+		}
+		if (entry.type == Entry::Type::CREATE)
+			dropped_databases.erase(entry.db);
+		return true;
+	}
+
+	DBUG_ASSERT(entry.type == Entry::Type::DROP);
+
+	std::string db_path(datadir_path);
+	db_path.append("/").append(entry.db);
+	const char	*dst_path = convert_dst(db_path.c_str());
+	if (!ds_remove(ds, dst_path)) {
+		dropped_databases.insert(entry.db);
+		return true;
+	}
+	return false;
+}
+
+static
+std::unique_ptr<std::vector<std::string>>
+	find_table_files(
+		const char *dir_path,
+		const std::string &db,
+		const std::string &table) {
+
+	std::unique_ptr<std::vector<std::string>>
+		result(new std::vector<std::string>());
+
+	std::string prefix = convert_tablename_to_filepath(dir_path, db, table);
+	foreach_file_in_db_dirs(dir_path, [&](const char *file_name)->bool {
+		if (!strncmp(file_name, prefix.c_str(), prefix.size())) {
+			DBUG_ASSERT(strlen(file_name) >= prefix.size());
+			if (file_name[prefix.size()] == '.' ||
+				!strncmp(file_name + prefix.size(), "#P#", strlen("#P#")))
+				result->push_back(std::string(file_name));
+		}
+		return true;
+	});
+
+	return result;
+}
+
+static
+bool process_remove(
+	const char *datadir_path,
+	ds_ctxt_t *ds,
+	const Entry &entry,
+	bool remove_frm) {
+
+	if (check_if_skip_table(
+		std::string(entry.db).append("/").append(entry.table).c_str()))
+		return true;
+
+	auto ext_it = engine_exts.find(entry.engine);
+	if (ext_it == engine_exts.end())
+		return true;
+
+	std::string file_preffix = convert_tablename_to_filepath(datadir_path,
+		entry.db, entry.table);
+	const char *dst_preffix = convert_dst(file_preffix.c_str());
+
+	for (const char *ext : ext_it->second) {
+		std::string old_name(dst_preffix);
+		if (!entry.partitioned)
+			old_name.append(ext);
+		else
+			old_name.append("#P#*");
+		if (ds_remove(ds, old_name.c_str())) {
+			msg("Failed to remove %s.", old_name.c_str());
+			return false;
+		}
+	}
+
+	if (remove_frm) {
+		std::string old_frm_name(dst_preffix);
+		old_frm_name.append(frm_ext);
+		if (ds_remove(ds, old_frm_name.c_str())) {
+			msg("Failed to remove %s.", old_frm_name.c_str());
+			return false;
+		}
+	}
+	return true;
+
+}
+
+static
+bool process_recopy(
+	const char *datadir_path,
+	ds_ctxt_t *ds,
+	const Entry &entry,
+	const tables_t &tables) {
+
+	if (check_if_skip_table(
+		std::string(entry.db).append("/").append(entry.table).c_str()))
+		return true;
+
+	const std::string &new_table_id =
+		entry.new_id.empty() ? entry.id : entry.new_id;
+	DBUG_ASSERT(!new_table_id.empty());
+	const std::string &new_table =
+		entry.new_table.empty() ? entry.table : entry.new_table;
+	DBUG_ASSERT(!new_table.empty());
+	const std::string &new_db =
+		entry.new_db.empty() ? entry.db : entry.new_db;
+	DBUG_ASSERT(!new_db.empty());
+	const std::string &new_engine =
+		entry.new_engine.empty() ? entry.engine : entry.new_engine;
+	DBUG_ASSERT(!new_engine.empty());
+
+	if (entry.type != Entry::Type::BULK_INSERT) {
+		auto table_it = tables.find(table_key(new_db, new_table));
+		if (table_it != tables.end() &&
+			table_it->second == new_table_id)
+			return true;
+	}
+
+	if (!entry.new_engine.empty() &&
+		entry.engine != entry.new_engine &&
+		!known_engine(entry.new_engine)) {
+		return process_remove(datadir_path, ds, entry, false);
+	}
+
+	if ((entry.partitioned || entry.new_partitioned) &&
+		!process_remove(datadir_path, ds, entry, false))
+		return false;
+
+	if (entry.partitioned || entry.new_partitioned) {
+		auto files = find_table_files(datadir_path, new_db, new_table);
+		if (!files.get())
+			return true;
+		for (const auto &file : *files) {
+			const char *dst_path = convert_dst(file.c_str());
+			if (!ds->copy_file(file.c_str(), dst_path, 0, true)) {
+				msg("Failed to re-copy %s.", file.c_str());
+				return false;
+			}
+		}
+		return true;
+	}
+
+	auto ext_it = engine_exts.find(new_engine);
+	if (ext_it == engine_exts.end())
+		return false;
+
+	for (const char *ext : ext_it->second) {
+		std::string file_name =
+			convert_tablename_to_filepath(datadir_path, new_db, new_table).
+				append(ext);
+		const char *dst_path = convert_dst(file_name.c_str());
+		if (file_exists(file_name.c_str()) &&
+			!ds->copy_file(file_name.c_str(), dst_path, 0, true)) {
+			msg("Failed to re-copy %s.", file_name.c_str());
+			return false;
+		}
+	}
+
+	std::string frm_file =
+		convert_tablename_to_filepath(datadir_path, new_db, new_table).
+			append(frm_ext);
+	const char *frm_dst_path = convert_dst(frm_file.c_str());
+	if (file_exists(frm_file.c_str()) &&
+		!ds->copy_file(frm_file.c_str(), frm_dst_path, 0, true)) {
+		msg("Failed to re-copy %s.", frm_file.c_str());
+		return false;
+	}
+
+	return true;
+}
+
+static
+bool process_rename(
+	const char *datadir_path,
+	ds_ctxt_t *ds,
+	const Entry &entry) {
+
+	if (check_if_skip_table(
+		std::string(entry.db).append("/").append(entry.table).c_str()))
+		return true;
+
+	DBUG_ASSERT(entry.db != "partition");
+
+	auto ext_it = engine_exts.find(entry.engine);
+	if (ext_it == engine_exts.end())
+		return false;
+
+	std::string new_preffix = convert_tablename_to_filepath(datadir_path,
+		entry.new_db, entry.new_table);
+	const char	*dst_path = convert_dst(new_preffix.c_str());
+
+	std::string old_preffix = convert_tablename_to_filepath(datadir_path,
+		entry.db, entry.table);
+	const char	*src_path = convert_dst(old_preffix.c_str());
+
+	for (const char *ext : ext_it->second) {
+		std::string old_name(src_path);
+		old_name.append(ext);
+		std::string new_name(dst_path);
+		new_name.append(ext);
+		if (ds_rename(ds, old_name.c_str(), new_name.c_str())) {
+			msg("Failed to rename %s to %s.",
+				old_name.c_str(), new_name.c_str());
+			return false;
+		}
+	}
+
+	std::string new_frm_file = new_preffix + frm_ext;
+	const char	*new_frm_dst = convert_dst(new_frm_file.c_str());
+	if (file_exists(new_frm_file.c_str()) &&
+		!ds->copy_file(new_frm_file.c_str(), new_frm_dst, 0, true)) {
+		msg("Failed to re-copy %s.", new_frm_file.c_str());
+		return false;
+	}
+
+// TODO: return this code if .frm is copied not under BLOCK_DDL
+/*
+	std::string old_frm_name(src_path);
+	old_frm_name.append(frm_ext);
+	std::string new_frm_name(dst_path);
+	new_frm_name.append(frm_ext);
+	if (ds_rename(ds, old_frm_name.c_str(), new_frm_name.c_str())) {
+		msg("Failed to rename %s to %s.",
+			old_frm_name.c_str(), new_frm_name.c_str());
+		return false;
+	}
+*/
+	return true;
+}
+
+bool backup(
+	const char *datadir_path,
+	ds_ctxt_t *ds,
+	const tables_t &tables) {
+	DBUG_ASSERT(datadir_path);
+	DBUG_ASSERT(ds);
+	char ddl_log_path[FN_REFLEN];
+	fn_format(ddl_log_path, "ddl", datadir_path, ".log", 0);
+	std::vector<std::unique_ptr<Entry>> entries;
+
+	std::unordered_set<std::string> processed_tables;
+	std::unordered_set<std::string> dropped_databases;
+
+	bool parsing_result =
+		parse(ddl_log_path, [&](std::unique_ptr<Entry> entry)->bool {
+
+			if (entry->engine == database_keyword)
+				return process_database(datadir_path, ds, *entry, dropped_databases);
+
+			if (!known_engine(entry->engine) && !known_engine(entry->new_engine))
+				return true;
+
+			if (entry->type == Entry::Type::CREATE ||
+					(entry->type == Entry::Type::ALTER &&
+					!entry->new_engine.empty() &&
+					entry->engine != entry->new_engine)) {
+				if (!process_recopy(datadir_path, ds, *entry, tables))
+					return false;
+				processed_tables.insert(table_key(entry->db, entry->table));
+				if (entry->type == Entry::Type::ALTER)
+					processed_tables.insert(table_key(entry->new_db, entry->new_table));
+				return true;
+			}
+
+			if (entry->type == Entry::Type::DROP) {
+				if (!process_remove(datadir_path, ds, *entry, true))
+					return false;
+				processed_tables.insert(table_key(entry->db, entry->table));
+				return true;
+			}
+			if (entry->type == Entry::Type::RENAME) {
+				if (entry->partitioned) {
+					if (!process_remove(datadir_path, ds, *entry, true))
+						return false;
+					Entry recopy_entry {
+						entry->type,
+						{},
+						entry->new_engine.empty() ? entry->engine : entry->new_engine,
+						true,
+						entry->new_db,
+						entry->new_table,
+						entry->new_id,
+						{}, true, {}, {}, {}
+					};
+					if (!process_recopy(datadir_path, ds, recopy_entry, tables))
+						return false;
+				}
+				else if (!process_rename(datadir_path, ds, *entry))
+					return false;
+				processed_tables.insert(table_key(entry->db, entry->table));
+				processed_tables.insert(table_key(entry->new_db, entry->new_table));
+				return true;
+			}
+
+			entries.push_back(std::move(entry));
+			return true;
+
+		});
+
+	if (!parsing_result)
+		return false;
+
+
+	while (!entries.empty()) {
+		auto entry = std::move(entries.back());
+		entries.pop_back();
+		auto tk = table_key(
+			entry->new_db.empty() ? entry->db : entry->new_db,
+			entry->new_table.empty() ? entry->table : entry->new_table);
+		if (dropped_databases.count(entry->db) ||
+			dropped_databases.count(entry->new_db))
+			continue;
+		if (processed_tables.count(tk))
+			continue;
+		processed_tables.insert(std::move(tk));
+		if (!process_recopy(datadir_path, ds, *entry, tables))
+			return false;
+	}
+
+	return true;
+}
+
+} // namespace ddl_log

extra/mariabackup/ddl_log.h

@@ -0,0 +1,15 @@
+#pragma once
+#include "my_global.h"
+#include "datasink.h"
+#include "aria_backup_client.h"
+#include <string>
+#include <memory>
+#include <vector>
+#include <unordered_map>
+
+namespace ddl_log {
+
+typedef std::unordered_map<std::string, std::string> tables_t;
+bool backup(const char *datadir_path, ds_ctxt_t *ds, const tables_t &tables);
+
+} // namespace ddl_log

extra/mariabackup/ds_buffer.cc

@@ -44,7 +44,7 @@ typedef struct {
 
 static ds_ctxt_t *buffer_init(const char *root);
 static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path,
-			      MY_STAT *mystat);
+			      const MY_STAT *mystat, bool rewrite);
 static int buffer_write(ds_file_t *file, const uchar *buf, size_t len);
 static int buffer_close(ds_file_t *file);
 static void buffer_deinit(ds_ctxt_t *ctxt);
@@ -53,8 +53,11 @@ datasink_t datasink_buffer = {
 	&buffer_init,
 	&buffer_open,
 	&buffer_write,
+	nullptr,
 	&buffer_close,
 	&dummy_remove,
+	nullptr,
+	nullptr,
 	&buffer_deinit
 };
 
@@ -84,8 +87,10 @@ buffer_init(const char *root)
 }
 
 static ds_file_t *
-buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+buffer_open(ds_ctxt_t *ctxt, const char *path,
+	const MY_STAT *mystat, bool rewrite)
 {
+  DBUG_ASSERT(rewrite == false);
 	ds_buffer_ctxt_t	*buffer_ctxt;
 	ds_ctxt_t		*pipe_ctxt;
 	ds_file_t		*dst_file;

extra/mariabackup/ds_compress.cc

@@ -65,7 +65,7 @@ extern ulonglong	xtrabackup_compress_chunk_size;
 
 static ds_ctxt_t *compress_init(const char *root);
 static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path,
-				MY_STAT *mystat);
+				const MY_STAT *mystat, bool rewrite);
 static int compress_write(ds_file_t *file, const uchar *buf, size_t len);
 static int compress_close(ds_file_t *file);
 static void compress_deinit(ds_ctxt_t *ctxt);
@@ -74,8 +74,11 @@ datasink_t datasink_compress = {
 	&compress_init,
 	&compress_open,
 	&compress_write,
+	nullptr,
 	&compress_close,
 	&dummy_remove,
+	nullptr,
+	nullptr,
 	&compress_deinit
 };
 
@@ -116,8 +119,10 @@ compress_init(const char *root)
 
 static
 ds_file_t *
-compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+compress_open(ds_ctxt_t *ctxt, const char *path,
+	const MY_STAT *mystat, bool rewrite)
 {
+  DBUG_ASSERT(rewrite == false);
 	ds_compress_ctxt_t	*comp_ctxt;
 	ds_ctxt_t		*dest_ctxt;
  	ds_file_t		*dest_file;

extra/mariabackup/ds_local.cc

@@ -42,8 +42,9 @@ typedef struct {
 
 static ds_ctxt_t *local_init(const char *root);
 static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path,
-			     MY_STAT *mystat);
+			     const MY_STAT *mystat, bool rewrite);
 static int local_write(ds_file_t *file, const uchar *buf, size_t len);
+static int local_seek_set(ds_file_t *file, my_off_t offset);
 static int local_close(ds_file_t *file);
 static void local_deinit(ds_ctxt_t *ctxt);
 
@@ -52,13 +53,20 @@ static int local_remove(const char *path)
 	return unlink(path);
 }
 
+static int local_rename(
+	ds_ctxt_t *ctxt, const char *old_path, const char *new_path);
+static int local_mremove(ds_ctxt_t *ctxt, const char *path);
+
 extern "C" {
 datasink_t datasink_local = {
 	&local_init,
 	&local_open,
 	&local_write,
+	&local_seek_set,
 	&local_close,
 	&local_remove,
+	&local_rename,
+	&local_mremove,
 	&local_deinit
 };
 }
@@ -89,7 +97,7 @@ local_init(const char *root)
 static
 ds_file_t *
 local_open(ds_ctxt_t *ctxt, const char *path,
-	   MY_STAT *mystat __attribute__((unused)))
+	   const MY_STAT *mystat __attribute__((unused)), bool rewrite)
 {
 	char 		fullpath[FN_REFLEN];
 	char		dirpath[FN_REFLEN];
@@ -111,7 +119,9 @@ local_open(ds_ctxt_t *ctxt, const char *path,
 		return NULL;
 	}
 
-	fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW,
+	// TODO: check in Windows and set the corresponding flags on fail
+	fd = my_create(fullpath, 0,
+		O_WRONLY | O_BINARY | (rewrite ? O_TRUNC : O_EXCL) | O_NOFOLLOW,
 		MYF(MY_WME));
 	if (fd < 0) {
 		return NULL;
@@ -194,8 +204,8 @@ static void init_ibd_data(ds_local_file_t *local_file, const uchar *buf, size_t
 		return;
 	}
 
-	auto flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]);
+	uint32_t flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]);
-	auto ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags);
+	uint32_t ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags);
 	local_file->pagesize= ssize == 0 ? UNIV_PAGE_SIZE_ORIG : ((UNIV_ZIP_SIZE_MIN >> 1) << ssize);
 	local_file->compressed = fil_space_t::full_crc32(flags)
 		? fil_space_t::is_compressed(flags)
@@ -239,6 +249,15 @@ local_write(ds_file_t *file, const uchar *buf, size_t len)
 	return 1;
 }
 
+static
+int
+local_seek_set(ds_file_t *file, my_off_t offset) {
+	ds_local_file_t *local_file= (ds_local_file_t *)file->ptr;
+	if (my_seek(local_file->fd, offset, SEEK_SET, MYF(0)) == MY_FILEPOS_ERROR)
+		return 1;
+	return 0;
+}
+
 /* Set EOF at file's current position.*/
 static int set_eof(File fd)
 {
@@ -276,3 +295,77 @@ local_deinit(ds_ctxt_t *ctxt)
 	my_free(ctxt->root);
 	my_free(ctxt);
 }
+
+
+static int local_rename(
+	ds_ctxt_t *ctxt, const char *old_path, const char *new_path) {
+	char		full_old_path[FN_REFLEN];
+	char		full_new_path[FN_REFLEN];
+	fn_format(full_old_path, old_path, ctxt->root, "", MYF(MY_RELATIVE_PATH));
+	fn_format(full_new_path, new_path, ctxt->root, "", MYF(MY_RELATIVE_PATH));
+	// Ignore errors as .frm files can me copied separately.
+	// TODO: return error processing here after the corresponding changes in
+	// xtrabackup.cc
+	(void)my_rename(full_old_path, full_new_path, MYF(0));
+//	if (my_rename(full_old_path, full_new_path, MYF(0))) {
+//		msg("Failed to rename file %s to %s", old_path, new_path);
+//		return 1;
+//	}
+	return 0;
+}
+
+// It's ok if destination does not contain the file or folder
+static int local_mremove(ds_ctxt_t *ctxt, const char *path) {
+	char		full_path[FN_REFLEN];
+	fn_format(full_path, path, ctxt->root, "", MYF(MY_RELATIVE_PATH));
+	size_t full_path_len = strlen(full_path);
+	if (full_path[full_path_len - 1] == '*') {
+		full_path[full_path_len - 1] = '\0';
+		char *preffix = strrchr(full_path, '/');
+		const char *full_path_dir = full_path;
+		size_t preffix_len;
+		if (preffix) {
+			preffix_len = (full_path_len - 1) - (preffix - full_path);
+			*(preffix++) = '\0';
+		}
+		else {
+			preffix = full_path;
+			preffix_len = full_path_len - 1;
+			full_path_dir= IF_WIN(".\\", "./");
+		}
+		if (!preffix_len)
+			return 0;
+		MY_DIR *dir= my_dir(full_path_dir, 0);
+		if (!dir)
+			return 0;
+		for (size_t i = 0; i < dir->number_of_files; ++i) {
+			char full_fpath[FN_REFLEN];
+			if (strncmp(dir->dir_entry[i].name, preffix, preffix_len))
+				continue;
+			fn_format(full_fpath, dir->dir_entry[i].name,
+					full_path_dir, "", MYF(MY_RELATIVE_PATH));
+			(void)my_delete(full_fpath, MYF(0));
+		}
+		my_dirend(dir);
+	}
+	else {
+		MY_STAT stat;
+		if (!my_stat(full_path, &stat, MYF(0)))
+			return 0;
+		MY_DIR *dir= my_dir(full_path, 0);
+		if (!dir) {
+			// TODO: check for error here if necessary
+			(void)my_delete(full_path, MYF(0));
+			return 0;
+		}
+		for (size_t i = 0; i < dir->number_of_files; ++i) {
+			char		full_fpath[FN_REFLEN];
+			fn_format(full_fpath, dir->dir_entry[i].name,
+					full_path, "", MYF(MY_RELATIVE_PATH));
+			(void)my_delete(full_fpath, MYF(0));
+		}
+		my_dirend(dir);
+		(void)my_rmtree(full_path, MYF(0));
+	}
+	return 0;
+}

extra/mariabackup/ds_stdout.cc

@@ -30,7 +30,7 @@ typedef struct {
 
 static ds_ctxt_t *stdout_init(const char *root);
 static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path,
-			     MY_STAT *mystat);
+			const MY_STAT *mystat, bool rewrite);
 static int stdout_write(ds_file_t *file, const uchar *buf, size_t len);
 static int stdout_close(ds_file_t *file);
 static void stdout_deinit(ds_ctxt_t *ctxt);
@@ -39,8 +39,11 @@ datasink_t datasink_stdout = {
 	&stdout_init,
 	&stdout_open,
 	&stdout_write,
+	nullptr,
 	&stdout_close,
 	&dummy_remove,
+	nullptr,
+	nullptr,
 	&stdout_deinit
 };
 
@@ -61,8 +64,9 @@ static
 ds_file_t *
 stdout_open(ds_ctxt_t *ctxt __attribute__((unused)),
 	    const char *path __attribute__((unused)),
-	    MY_STAT *mystat __attribute__((unused)))
+	    const MY_STAT *mystat __attribute__((unused)), bool rewrite)
 {
+  DBUG_ASSERT(rewrite == false);
 	ds_stdout_file_t 	*stdout_file;
 	ds_file_t		*file;
 	size_t			pathlen;

extra/mariabackup/ds_tmpfile.cc

@@ -41,7 +41,7 @@ typedef struct {
 
 static ds_ctxt_t *tmpfile_init(const char *root);
 static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path,
-			       MY_STAT *mystat);
+			const MY_STAT *mystat, bool rewrite);
 static int tmpfile_write(ds_file_t *file, const uchar *buf, size_t len);
 static int tmpfile_close(ds_file_t *file);
 static void tmpfile_deinit(ds_ctxt_t *ctxt);
@@ -50,8 +50,11 @@ datasink_t datasink_tmpfile = {
 	&tmpfile_init,
 	&tmpfile_open,
 	&tmpfile_write,
+	nullptr,
 	&tmpfile_close,
 	&dummy_remove,
+	nullptr,
+	nullptr,
 	&tmpfile_deinit
 };
 
@@ -80,8 +83,9 @@ tmpfile_init(const char *root)
 
 static ds_file_t *
 tmpfile_open(ds_ctxt_t *ctxt, const char *path,
-			       MY_STAT *mystat)
+			const MY_STAT *mystat, bool rewrite)
 {
+  DBUG_ASSERT(rewrite == false);
 	ds_tmpfile_ctxt_t	*tmpfile_ctxt;
 	char			 tmp_path[FN_REFLEN];
 	ds_tmp_file_t		*tmp_file;

extra/mariabackup/ds_xbstream.cc

@@ -40,24 +40,31 @@ General streaming interface */
 
 static ds_ctxt_t *xbstream_init(const char *root);
 static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path,
-			      MY_STAT *mystat);
+			const MY_STAT *mystat, bool rewrite);
 static int xbstream_write(ds_file_t *file, const uchar *buf, size_t len);
+static int xbstream_seek_set(ds_file_t *file, my_off_t offset);
 static int xbstream_close(ds_file_t *file);
 static void xbstream_deinit(ds_ctxt_t *ctxt);
 
+static int xbstream_rename(
+	ds_ctxt_t *ctxt, const char *old_path, const char *new_path);
+static int xbstream_mremove(ds_ctxt_t *ctxt, const char *path);
+
 datasink_t datasink_xbstream = {
 	&xbstream_init,
 	&xbstream_open,
 	&xbstream_write,
+	&xbstream_seek_set,
 	&xbstream_close,
 	&dummy_remove,
+	&xbstream_rename,
+	&xbstream_mremove,
 	&xbstream_deinit
 };
 
 static
 ssize_t
-my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)),
+my_xbstream_write_callback(void *userdata, const void *buf, size_t len)
-		       void *userdata, const void *buf, size_t len)
 {
 	ds_stream_ctxt_t	*stream_ctxt;
 
@@ -89,7 +96,7 @@ xbstream_init(const char *root __attribute__((unused)))
 		goto err;
 	}
 
-	xbstream = xb_stream_write_new();
+	xbstream = xb_stream_write_new(my_xbstream_write_callback, stream_ctxt);
 	if (xbstream == NULL) {
 		msg("xb_stream_write_new() failed.");
 		goto err;
@@ -108,7 +115,8 @@ err:
 
 static
 ds_file_t *
-xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+xbstream_open(ds_ctxt_t *ctxt, const char *path,
+	const MY_STAT *mystat, bool rewrite)
 {
 	ds_file_t		*file;
 	ds_stream_file_t	*stream_file;
@@ -144,9 +152,7 @@ xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
 
 	xbstream = stream_ctxt->xbstream;
 
-	xbstream_file = xb_stream_write_open(xbstream, path, mystat,
+	xbstream_file = xb_stream_write_open(xbstream, path, mystat, rewrite);
-		                             stream_ctxt,
-					     my_xbstream_write_callback);
 
 	if (xbstream_file == NULL) {
 		msg("xb_stream_write_open() failed.");
@@ -190,6 +196,45 @@ xbstream_write(ds_file_t *file, const uchar *buf, size_t len)
 	return 0;
 }
 
+static
+int
+xbstream_seek_set(ds_file_t *file, my_off_t offset)
+{
+	ds_stream_file_t	*stream_file;
+	xb_wstream_file_t	*xbstream_file;
+
+
+	stream_file = (ds_stream_file_t *) file->ptr;
+
+	xbstream_file = stream_file->xbstream_file;
+
+	if (xb_stream_write_seek_set(xbstream_file, offset)) {
+		msg("xb_stream_write_seek_set() failed.");
+		return 1;
+	}
+
+	return 0;
+}
+
+static
+int
+xbstream_mremove(ds_ctxt_t *ctxt, const char *path) {
+	ds_stream_ctxt_t	*stream_ctxt =
+		reinterpret_cast<ds_stream_ctxt_t *>(ctxt->ptr);
+	xb_wstream_t		*xbstream = stream_ctxt->xbstream;
+	return xb_stream_write_remove(xbstream, path);
+}
+
+static
+int
+xbstream_rename(
+	ds_ctxt_t *ctxt, const char *old_path, const char *new_path) {
+	ds_stream_ctxt_t	*stream_ctxt =
+		reinterpret_cast<ds_stream_ctxt_t *>(ctxt->ptr);
+	xb_wstream_t		*xbstream = stream_ctxt->xbstream;
+	return xb_stream_write_rename(xbstream, old_path, new_path);
+}
+
 static
 int
 xbstream_close(ds_file_t *file)

extra/mariabackup/encryption_plugin.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2017, 2022, MariaDB Corporation.
+/* Copyright (c) 2017, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -17,18 +17,18 @@
 #include <mysqld.h>
 #include <mysql.h>
 #include <xtrabackup.h>
-#include <xb_plugin.h>
+#include <encryption_plugin.h>
 #include <sql_plugin.h>
 #include <sstream>
 #include <vector>
 #include <common.h>
 #include <backup_mysql.h>
-#include <srv0srv.h>
+#include <log0crypt.h>
 
 
 extern struct st_maria_plugin *mysql_optional_plugins[];
 extern struct st_maria_plugin *mysql_mandatory_plugins[];
-static void xb_plugin_init(int argc, char **argv);
+static void encryption_plugin_init(int argc, char **argv);
 
 extern char *xb_plugin_load;
 extern char *xb_plugin_dir;
@@ -42,7 +42,7 @@ const char *QUERY_PLUGIN =
 " OR (plugin_type = 'DAEMON' AND plugin_name LIKE 'provider\\_%')"
 " AND plugin_status='ACTIVE'";
 
-std::string xb_plugin_config;
+std::string encryption_plugin_config;
 
 static void add_to_plugin_load_list(const char *plugin_def)
 {
@@ -52,16 +52,16 @@ static void add_to_plugin_load_list(const char *plugin_def)
 static char XTRABACKUP_EXE[] = "xtrabackup";
 
 /*
-  Read "plugin-load" value from backup-my.cnf during prepare phase.
+  Read "plugin-load" value (encryption plugin) from backup-my.cnf during
+  prepare phase.
   The value is stored during backup phase.
 */
-static std::string get_plugin_from_cnf(const char *dir)
+static std::string get_encryption_plugin_from_cnf()
 {
-  std::string path = dir + std::string("/backup-my.cnf");
+  FILE *f = fopen("backup-my.cnf", "r");
-  FILE *f = fopen(path.c_str(), "r");
   if (!f)
   {
-    die("Can't open %s for reading", path.c_str());
+    die("Can't open backup-my.cnf for reading");
   }
   char line[512];
   std::string plugin_load;
@@ -72,7 +72,16 @@ static std::string get_plugin_from_cnf(const char *dir)
       plugin_load = line + 12;
       // remote \n at the end of string
       plugin_load.resize(plugin_load.size() - 1);
-      break;
+    }
+
+    if (strncmp(line, "innodb_encrypt_tables=", 22) == 0)
+    {
+      if (!strncmp(line + 22, "ON", 2) ||
+          !strncmp(line + 22, "1", 1))
+        srv_encrypt_tables= 1;
+      else if (!strncmp(line + 22, "FORCE", 5) ||
+               !strncmp(line + 22, "2", 1))
+        srv_encrypt_tables= 2;
     }
   }
   fclose(f);
@@ -80,7 +89,7 @@ static std::string get_plugin_from_cnf(const char *dir)
 }
 
 
-void xb_plugin_backup_init(MYSQL *mysql)
+void encryption_plugin_backup_init(MYSQL *mysql)
 {
   MYSQL_RES *result;
   MYSQL_ROW row;
@@ -163,7 +172,18 @@ void xb_plugin_backup_init(MYSQL *mysql)
     mysql_free_result(result);
   }
 
-  xb_plugin_config = oss.str();
+  result = xb_mysql_query(mysql, "select @@innodb_encrypt_tables", true, true);
+  row = mysql_fetch_row(result);
+  if (!row);
+  else if (const char *r= row[0])
+  {
+    if (!strcmp(r, "ON")) srv_encrypt_tables= 1;
+    else if (!strcmp(r, "FORCE")) srv_encrypt_tables= 2;
+    oss << "innodb_encrypt_tables=" << r << std::endl;
+  }
+
+  mysql_free_result(result);
+  encryption_plugin_config = oss.str();
 
   argc = 0;
   argv[argc++] = XTRABACKUP_EXE;
@@ -175,23 +195,23 @@ void xb_plugin_backup_init(MYSQL *mysql)
   }
   argv[argc] = 0;
 
-  xb_plugin_init(argc, argv);
+  encryption_plugin_init(argc, argv);
 }
 
-const char *xb_plugin_get_config()
+const char *encryption_plugin_get_config()
 {
-  return xb_plugin_config.c_str();
+  return encryption_plugin_config.c_str();
 }
 
 extern int finalize_encryption_plugin(st_plugin_int *plugin);
 
 
-void xb_plugin_prepare_init(int argc, char **argv, const char *dir)
+void encryption_plugin_prepare_init(int argc, char **argv)
 {
-  std::string plugin_load= get_plugin_from_cnf(dir ? dir : ".");
+  std::string plugin_load= get_encryption_plugin_from_cnf();
   if (plugin_load.size())
   {
-    msg("Loading plugins from %s", plugin_load.c_str());
+    msg("Loading encryption plugin from %s", plugin_load.c_str());
   }
   else
   {
@@ -211,19 +231,19 @@ void xb_plugin_prepare_init(int argc, char **argv, const char *dir)
   new_argv[0] = XTRABACKUP_EXE;
   memcpy(&new_argv[1], argv, argc*sizeof(char *));
 
-  xb_plugin_init(argc+1, new_argv);
+  encryption_plugin_init(argc+1, new_argv);
 
   delete[] new_argv;
 }
 
-static void xb_plugin_init(int argc, char **argv)
+static void encryption_plugin_init(int argc, char **argv)
 {
   /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */
   mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0;
   plugin_maturity = MariaDB_PLUGIN_MATURITY_UNKNOWN; /* mariabackup accepts all plugins */
-  msg("Loading plugins");
+  msg("Loading encryption plugin");
   for (int i= 1; i < argc; i++)
-    msg("\t Plugin parameter :  '%s'", argv[i]);
+    msg("\t Encryption plugin parameter :  '%s'", argv[i]);
   plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE);
 }
 

extra/mariabackup/encryption_plugin.h

@@ -0,0 +1,7 @@
+#include <mysql.h>
+#include <string>
+extern void encryption_plugin_backup_init(MYSQL *mysql);
+extern const char* encryption_plugin_get_config();
+extern void encryption_plugin_prepare_init(int argc, char **argv);
+
+//extern  void encryption_plugin_init(int argc, char **argv);

extra/mariabackup/fil_cur.cc

@@ -231,12 +231,14 @@ xb_fil_cur_open(
 				      / cursor->page_size);
 
 	cursor->read_filter = read_filter;
-	cursor->read_filter->init(&cursor->read_filter_ctxt, cursor,
+	cursor->read_filter->init(&cursor->read_filter_ctxt, cursor);
-				  node->space->id);
 
 	return(XB_FIL_CUR_SUCCESS);
 }
 
+/* Stack usage 131224 with clang */
+PRAGMA_DISABLE_CHECK_STACK_FRAME
+
 static bool page_is_corrupted(const byte *page, ulint page_no,
 			      const xb_fil_cur_t *cursor,
 			      const fil_space_t *space)
@@ -340,6 +342,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no,
 
 	return buf_page_is_corrupted(true, page, space->flags);
 }
+PRAGMA_REENABLE_CHECK_STACK_FRAME
 
 /** Reads and verifies the next block of pages from the source
 file. Positions the cursor after the last read non-corrupted page.
@@ -502,10 +505,6 @@ xb_fil_cur_close(
 /*=============*/
 	xb_fil_cur_t *cursor)	/*!< in/out: source file cursor */
 {
-	if (cursor->read_filter) {
-		cursor->read_filter->deinit(&cursor->read_filter_ctxt);
-	}
-
 	aligned_free(cursor->buf);
 	cursor->buf = NULL;
 

extra/mariabackup/fil_cur.h

@@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 
 #include <my_dir.h>
 #include "read_filt.h"
+#include "mtr0types.h"
 #include "srv0start.h"
 #include "srv0srv.h"
 #include "xtrabackup.h"

extra/mariabackup/innobackupex.cc

@@ -78,10 +78,8 @@ my_bool opt_ibx_galera_info = FALSE;
 my_bool opt_ibx_slave_info = FALSE;
 my_bool opt_ibx_no_lock = FALSE;
 my_bool opt_ibx_safe_slave_backup = FALSE;
-my_bool opt_ibx_rsync = FALSE;
 my_bool opt_ibx_force_non_empty_dirs = FALSE;
 my_bool opt_ibx_noversioncheck = FALSE;
-my_bool opt_ibx_no_backup_locks = FALSE;
 my_bool opt_ibx_decompress = FALSE;
 
 char *opt_ibx_incremental_history_name = NULL;
@@ -268,8 +266,10 @@ static struct my_option ibx_long_options[] =
 	 (uchar *) &opt_ibx_incremental, (uchar *) &opt_ibx_incremental, 0,
 	 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 
-	{"no-lock", OPT_NO_LOCK, "Use this option to disable table lock "
+	{"no-lock", OPT_NO_LOCK, "This option should not be used as "
-	 "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your "
+         "mariadb-backup now is using BACKUP LOCKS, which minimizes the "
+         "lock time. ALTER TABLE can run in parallel with BACKUP LOCKS."
+         "Use the --no-lock option it only if ALL your "
 	 "tables are InnoDB and you DO NOT CARE about the binary log "
 	 "position of the backup. This option shouldn't be used if there "
 	 "are any DDL statements being executed or if any updates are "
@@ -297,15 +297,6 @@ static struct my_option ibx_long_options[] =
 	 (uchar *) &opt_ibx_safe_slave_backup,
 	 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 
-	{"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file "
-	 "transfers. When this option is specified, innobackupex uses rsync "
-	 "to copy all non-InnoDB files instead of spawning a separate cp for "
-	 "each file, which can be much faster for servers with a large number "
-	 "of databases or tables.  This option cannot be used together with "
-	 "--stream.",
-	 (uchar *) &opt_ibx_rsync, (uchar *) &opt_ibx_rsync,
-	 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-
 	{"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This "
 	 "option, when specified, makes --copy-back or --move-back transfer "
 	 "files to non-empty directories. Note that no existing files will be "
@@ -330,13 +321,9 @@ static struct my_option ibx_long_options[] =
 	 (uchar *) &opt_ibx_noversioncheck,
 	 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 
-	{"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if "
+	{"no-backup-locks", OPT_NO_BACKUP_LOCKS,
-	 "backup locks should be used instead of FLUSH TABLES WITH READ LOCK "
+         "Old disabled option which has no effect anymore.",
-	 "on the backup stage. The option has no effect when backup locks are "
+	 (uchar *) 0, (uchar*) 0,
-	 "not supported by the server. This option is enabled by default, "
-	 "disable with --no-backup-locks.",
-	 (uchar *) &opt_ibx_no_backup_locks,
-	 (uchar *) &opt_ibx_no_backup_locks,
 	 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 
 	{"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp "
@@ -402,11 +389,10 @@ static struct my_option ibx_long_options[] =
 	 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 
 	{"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE,
-	 "This option specifies which types of queries are allowed to complete "
+         "Old disabled option which has no effect anymore (not needed "
-	 "before innobackupex will issue the global lock. Default is all.",
+         "with BACKUP LOCKS)",
-	 (uchar*) &opt_ibx_lock_wait_query_type,
+	 (uchar*) 0, (uchar*) 0, &query_type_typelib, GET_ENUM,
-	 (uchar*) &opt_ibx_lock_wait_query_type, &query_type_typelib,
+         REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0},
-	 GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0},
 
 	{"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE,
 	 "This option specifies which types of queries should be killed to "
@@ -447,32 +433,32 @@ static struct my_option ibx_long_options[] =
 	 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 
 	{"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT,
-	 "This option specifies the number of seconds innobackupex waits "
+         "Old disabled option which has no effect anymore (not needed "
-	 "between starting FLUSH TABLES WITH READ LOCK and killing those "
+         "with BACKUP LOCKS)",
-	 "queries that block it. Default is 0 seconds, which means "
+	 (uchar*) 0, (uchar*) 0,  0, GET_UINT,
-	 "innobackupex will not attempt to kill any queries.",
-	 (uchar*) &opt_ibx_kill_long_queries_timeout,
-	 (uchar*) &opt_ibx_kill_long_queries_timeout, 0, GET_UINT,
 	 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 
 	{"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT,
-	 "This option specifies time in seconds that innobackupex should wait "
+         "Alias for startup-wait-timeout",
-	 "for queries that would block FTWRL before running it. If there are "
+         (uchar*) &opt_ibx_lock_wait_timeout,
-	 "still such queries when the timeout expires, innobackupex terminates "
+         (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT,
-	 "with an error. Default is 0, in which case innobackupex does not "
+	 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-	 "wait for queries to complete and starts FTWRL immediately.",
+
+	{"startup-wait-timeout", OPT_LOCK_WAIT_TIMEOUT,
+         "This option specifies time in seconds that mariadb-backup should wait for "
+         "BACKUP STAGE START to complete. BACKUP STAGE START has to wait until all "
+         "currently running queries using explicite LOCK TABLES has ended. "
+         "If there are still such queries when the timeout expires, mariadb-backup "
+         "terminates with an error. Default is 0, in which case mariadb-backup waits "
+         "indefinitely for BACKUP STAGE START to finish",
          (uchar*) &opt_ibx_lock_wait_timeout,
          (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT,
 	 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 
 	{"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD,
-	 "This option specifies the query run time threshold which is used by "
+         "Old disabled option which has no effect anymore (not needed "
-	 "innobackupex to detect long-running queries with a non-zero value "
+         "with BACKUP LOCKS)",
-	 "of --ftwrl-wait-timeout. FTWRL is not started until such "
+	 (uchar*) 0, (uchar*) 0,  0, GET_UINT,
-	 "long-running queries exist. This option has no effect if "
-	 "--ftwrl-wait-timeout is 0. Default value is 60 seconds.",
-	 (uchar*) &opt_ibx_lock_wait_threshold,
-	 (uchar*) &opt_ibx_lock_wait_threshold, 0, GET_UINT,
 	 REQUIRED_ARG, 60, 0, 0, 0, 0, 0},
 
 	{"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
@@ -864,10 +850,8 @@ ibx_init()
 	opt_slave_info = opt_ibx_slave_info;
 	opt_no_lock = opt_ibx_no_lock;
 	opt_safe_slave_backup = opt_ibx_safe_slave_backup;
-	opt_rsync = opt_ibx_rsync;
 	opt_force_non_empty_dirs = opt_ibx_force_non_empty_dirs;
 	opt_noversioncheck = opt_ibx_noversioncheck;
-	opt_no_backup_locks = opt_ibx_no_backup_locks;
 	opt_decompress = opt_ibx_decompress;
 
 	opt_incremental_history_name = opt_ibx_incremental_history_name;

extra/mariabackup/read_filt.cc

@@ -32,29 +32,13 @@ Perform read filter context initialization that is common to all read
 filters.  */
 static
 void
-common_init(
+rf_pass_through_init(
-/*========*/
 	xb_read_filt_ctxt_t*	ctxt,	/*!<in/out: read filter context */
 	const xb_fil_cur_t*	cursor)	/*!<in: file cursor */
 {
 	ctxt->offset = 0;
 	ctxt->data_file_size = cursor->statinfo.st_size;
 	ctxt->buffer_capacity = cursor->buf_size;
-	ctxt->page_size = cursor->page_size;
-}
-
-/****************************************************************//**
-Initialize the pass-through read filter. */
-static
-void
-rf_pass_through_init(
-/*=================*/
-	xb_read_filt_ctxt_t*	ctxt,	/*!<in/out: read filter context */
-	const xb_fil_cur_t*	cursor,	/*!<in: file cursor */
-	ulint			space_id __attribute__((unused)))
-					/*!<in: space id we are reading */
-{
-	common_init(ctxt, cursor);
 }
 
 /****************************************************************//**
@@ -65,143 +49,25 @@ rf_pass_through_get_next_batch(
 /*===========================*/
 	xb_read_filt_ctxt_t*	ctxt,			/*!<in/out: read filter
 							context */
-	ib_int64_t*		read_batch_start,	/*!<out: starting read
+	int64_t*		read_batch_start,	/*!<out: starting read
 							offset in bytes for the
 							next batch of pages */
-	ib_int64_t*		read_batch_len)		/*!<out: length in
+	int64_t*		read_batch_len)		/*!<out: length in
 							bytes of the next batch
 							of pages */
 {
 	*read_batch_start = ctxt->offset;
 	*read_batch_len = ctxt->data_file_size - ctxt->offset;
 
-	if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) {
+	if (*read_batch_len > (int64_t)ctxt->buffer_capacity) {
 		*read_batch_len = ctxt->buffer_capacity;
 	}
 
 	ctxt->offset += *read_batch_len;
 }
 
-/****************************************************************//**
-Deinitialize the pass-through read filter.  */
-static
-void
-rf_pass_through_deinit(
-/*===================*/
-	xb_read_filt_ctxt_t*	ctxt __attribute__((unused)))
-					/*!<in: read filter context */
-{
-}
-
-/****************************************************************//**
-Initialize the changed page bitmap-based read filter.  Assumes that
-the bitmap is already set up in changed_page_bitmap.  */
-static
-void
-rf_bitmap_init(
-/*===========*/
-	xb_read_filt_ctxt_t*	ctxt,		/*!<in/out: read filter
-						context */
-	const xb_fil_cur_t*	cursor,		/*!<in: read cursor */
-	ulint			space_id)	/*!<in: space id  */
-{
-	common_init(ctxt, cursor);
-	ctxt->bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap,
-						       space_id);
-	ctxt->filter_batch_end = 0;
-}
-
-/****************************************************************//**
-Get the next batch of pages for the bitmap read filter.  */
-static
-void
-rf_bitmap_get_next_batch(
-/*=====================*/
-	xb_read_filt_ctxt_t*	ctxt,			/*!<in/out: read filter
-							context */
-	ib_int64_t*		read_batch_start,	/*!<out: starting read
-							offset in bytes for the
-							next batch of pages */
-	ib_int64_t*		read_batch_len)		/*!<out: length in
-							bytes of the next batch
-							of pages */
-{
-	ulint	start_page_id;
-	const ulint	page_size	= ctxt->page_size;
-
-	start_page_id = (ulint)(ctxt->offset / page_size);
-
-	xb_a (ctxt->offset % page_size == 0);
-
-	if (start_page_id == ctxt->filter_batch_end) {
-
-		/* Used up all the previous bitmap range, get some more */
-		ulint next_page_id;
-
-		/* Find the next changed page using the bitmap */
-		next_page_id = xb_page_bitmap_range_get_next_bit
-			(ctxt->bitmap_range, TRUE);
-
-		if (next_page_id == ULINT_UNDEFINED) {
-			*read_batch_len = 0;
-			return;
-		}
-
-		ctxt->offset = next_page_id * page_size;
-
-		/* Find the end of the current changed page block by searching
-		for the next cleared bitmap bit */
-		ctxt->filter_batch_end
-			= xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range,
-							    FALSE);
-		xb_a(next_page_id < ctxt->filter_batch_end);
-	}
-
-	*read_batch_start = ctxt->offset;
-	if (ctxt->filter_batch_end == ULINT_UNDEFINED) {
-		/* No more cleared bits in the bitmap, need to copy all the
-		remaining pages.  */
-		*read_batch_len = ctxt->data_file_size - ctxt->offset;
-	} else {
-		*read_batch_len = ctxt->filter_batch_end * page_size
-			- ctxt->offset;
-	}
-
-	/* If the page block is larger than the buffer capacity, limit it to
-	buffer capacity.  The subsequent invocations will continue returning
-	the current block in buffer-sized pieces until ctxt->filter_batch_end
-	is reached, trigerring the next bitmap query.  */
-	if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) {
-		*read_batch_len = ctxt->buffer_capacity;
-	}
-
-	ctxt->offset += *read_batch_len;
-	xb_a (ctxt->offset % page_size == 0);
-	xb_a (*read_batch_start % page_size == 0);
-	xb_a (*read_batch_len % page_size == 0);
-}
-
-/****************************************************************//**
-Deinitialize the changed page bitmap-based read filter.  */
-static
-void
-rf_bitmap_deinit(
-/*=============*/
-	xb_read_filt_ctxt_t*	ctxt)	/*!<in/out: read filter context */
-{
-	xb_page_bitmap_range_deinit(ctxt->bitmap_range);
-}
-
 /* The pass-through read filter */
 xb_read_filt_t rf_pass_through = {
 	&rf_pass_through_init,
 	&rf_pass_through_get_next_batch,
-	&rf_pass_through_deinit
-};
-
-/* The changed page bitmap-based read filter */
-xb_read_filt_t rf_bitmap = {
-	&rf_bitmap_init,
-	&rf_bitmap_get_next_batch,
-	&rf_bitmap_deinit
 };

extra/mariabackup/read_filt.h

@@ -25,42 +25,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 #ifndef XB_READ_FILT_H
 #define XB_READ_FILT_H
 
-#include "changed_page_bitmap.h"
+#include <cstdint>
-
+#include <cstddef>
-typedef uint32_t space_id_t;
 
 struct xb_fil_cur_t;
 
 /* The read filter context */
 struct xb_read_filt_ctxt_t {
-	ib_int64_t		offset;		/*!< current file offset */
+	int64_t		offset;		/*!< current file offset */
-	ib_int64_t		data_file_size;	/*!< data file size */
+	int64_t		data_file_size;	/*!< data file size */
 	size_t		buffer_capacity;/*!< read buffer capacity */
-	space_id_t	space_id;	/*!< space id */
-	/* The following fields used only in bitmap filter */
-	/* Move these to union if any other filters are added in future */
-	xb_page_bitmap_range	*bitmap_range;	/*!< changed page bitmap range
-						iterator for space_id */
-	ulint		page_size;		/*!< page size */
-	ulint		filter_batch_end;/*!< the ending page id of the
-						 current changed page block in
-						 the bitmap */
-	/** TODO: remove this default constructor */
-	xb_read_filt_ctxt_t() : page_size(0) {}
 };
 
 /* The read filter */
 struct xb_read_filt_t {
 	void (*init)(xb_read_filt_ctxt_t* ctxt,
-		     const xb_fil_cur_t* cursor,
+		     const xb_fil_cur_t* cursor);
-		     ulint space_id);
 	void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt,
-			       ib_int64_t* read_batch_start,
+			       int64_t* read_batch_start,
-			       ib_int64_t* read_batch_len);
+			       int64_t* read_batch_len);
-	void (*deinit)(xb_read_filt_ctxt_t* ctxt);
 };
 
 extern xb_read_filt_t rf_pass_through;
-extern xb_read_filt_t rf_bitmap;
 
 #endif

extra/mariabackup/thread_pool.cc

@@ -0,0 +1,50 @@
+#include "thread_pool.h"
+#include "common.h"
+
+bool ThreadPool::start(size_t threads_count) {
+	if (!m_stopped)
+		return false;
+	m_stopped = false;
+	for (unsigned i = 0; i < threads_count; ++i)
+		m_threads.emplace_back(&ThreadPool::thread_func, this, i);
+	return true;
+}
+
+void ThreadPool::stop() {
+	if (m_stopped)
+		return;
+	m_stop = true;
+	m_cv.notify_all();
+	for (auto &t : m_threads)
+		t.join();
+	m_stopped = true;
+};
+
+void ThreadPool::push(ThreadPool::job_t &&j) {
+	std::unique_lock<std::mutex> lock(m_mutex);
+	m_jobs.push(j);
+	lock.unlock();
+	m_cv.notify_one();
+}
+
+void ThreadPool::thread_func(unsigned thread_num) {
+	if (my_thread_init())
+		die("Can't init mysql thread");
+	std::unique_lock<std::mutex> lock(m_mutex);
+	while(true) {
+		if (m_stop)
+			goto exit;
+		while (!m_jobs.empty()) {
+			if (m_stop)
+				goto exit;
+			job_t j = std::move(m_jobs.front());
+			m_jobs.pop();
+			lock.unlock();
+			j(thread_num);
+			lock.lock();
+		}
+		m_cv.wait(lock, [&] { return m_stop || !m_jobs.empty(); });
+	}
+exit:
+	my_thread_end();
+}

extra/mariabackup/thread_pool.h

@@ -0,0 +1,62 @@
+#pragma once
+#include <queue>
+#include <vector>
+#include <functional>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include "trx0sys.h"
+
+class ThreadPool {
+public:
+	typedef std::function<void(unsigned)> job_t;
+
+	ThreadPool() { m_stop = false; m_stopped = true; }
+	ThreadPool (ThreadPool &&other) = delete;
+	ThreadPool &  operator= (ThreadPool &&other) = delete;
+	ThreadPool(const ThreadPool &) = delete;
+	ThreadPool & operator= (const ThreadPool &) = delete;
+
+	bool start(size_t threads_count);
+	void stop();
+	void push(job_t &&j);
+	size_t threads_count() const { return m_threads.size(); }
+private:
+	void thread_func(unsigned thread_num);
+	std::mutex m_mutex;
+	std::condition_variable m_cv;
+	std::queue<job_t> m_jobs;
+	std::atomic<bool> m_stop;
+	std::atomic<bool> m_stopped;
+	std::vector<std::thread> m_threads;
+};
+
+class TasksGroup {
+public:
+	TasksGroup(ThreadPool &thread_pool) : m_thread_pool(thread_pool) {
+		m_tasks_count = 0;
+		m_tasks_result = 1;
+	}
+	void push_task(ThreadPool::job_t &&j) {
+		++m_tasks_count;
+		m_thread_pool.push(std::forward<ThreadPool::job_t>(j));
+	}
+	void finish_task(int res) {
+		--m_tasks_count;
+		m_tasks_result.fetch_and(res);
+	}
+	int get_result() const { return m_tasks_result; }
+	bool is_finished() const {
+		return !m_tasks_count;
+	}
+	bool wait_for_finish() {
+		while (!is_finished())
+                        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+		return get_result();
+	}
+private:
+	ThreadPool &m_thread_pool;
+	std::atomic<size_t> m_tasks_count;
+	std::atomic<int> m_tasks_result;
+};

extra/mariabackup/write_filt.cc

@@ -144,6 +144,18 @@ wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
 			return false;
 		}
 
+		/* Check whether TRX_SYS page has been changed */
+		if (mach_read_from_4(page + FIL_PAGE_SPACE_ID)
+				== TRX_SYS_SPACE
+		    && mach_read_from_4(page + FIL_PAGE_OFFSET)
+				== TRX_SYS_PAGE_NO) {
+			msg(cursor->thread_n,
+			    "--incremental backup is impossible if "
+			    "the server had been restarted with "
+			    "different innodb_undo_tablespaces.");
+			return false;
+		}
+
 		/* updated page */
 		if (cp->npages == page_size / 4) {
 			/* flush buffer */

extra/mariabackup/wsrep.cc

@@ -52,10 +52,12 @@ permission notice:
 #include <wsrep_api.h>
 
 /*! Name of file where Galera info is stored on recovery */
-#define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info"
 #define MB_GALERA_INFO_FILENAME "mariadb_backup_galera_info"
 #define XB_GALERA_DONOR_INFO_FILENAME "donor_galera_info"
 
+/* backup copy of galera info file as sent by donor */
+#define MB_GALERA_INFO_FILENAME_SST "mariadb_backup_galera_info_SST"
+
 /***********************************************************************
 Store Galera checkpoint info in the MB_GALERA_INFO_FILENAME file, if that
 information is present in the trx system header. Otherwise, do nothing. */
@@ -69,20 +71,45 @@ xb_write_galera_info(bool incremental_prepare)
 	long long	seqno;
 	MY_STAT		statinfo;
 
-	/* Do not overwrite an existing file to be compatible with
+	xid.null();
-	servers with older server versions */
-	if (!incremental_prepare &&
-		(my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL ||
-		 my_stat(MB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL)) {
 
+	/* try to read last wsrep XID from innodb rsegs, we will use it
+	   instead of galera info file received from donor
+	*/
+	if (!trx_rseg_read_wsrep_checkpoint(xid)) {
+		/* no worries yet, SST may have brought in galera info file
+		   from some old MariaDB version, which does not support
+		   wsrep XID storing in innodb rsegs
+		*/
 		return;
 	}
 
-	xid.null();
+	/* if SST brought in galera info file, copy it as *_SST file
+	   this will not be used, saved just for future reference
+	*/
+	if (my_stat(MB_GALERA_INFO_FILENAME, &statinfo, MYF(0))) {
+		FILE* fp_in  = fopen(MB_GALERA_INFO_FILENAME, "r");
+		FILE* fp_out = fopen(MB_GALERA_INFO_FILENAME_SST, "w");
 
-	if (!trx_rseg_read_wsrep_checkpoint(xid)) {
+		char buf[BUFSIZ] = {'\0'};
-
+		size_t size;
-		return;
+		while ((size = fread(buf, 1, BUFSIZ, fp_in))) {
+			if (fwrite(buf, 1, size, fp_out) != strlen(buf)) {
+				die(
+				    "could not write to "
+				    MB_GALERA_INFO_FILENAME_SST
+				    ", errno = %d\n",
+				    errno);
+			}
+		}
+		if (!feof(fp_in)) {
+			die(
+			    MB_GALERA_INFO_FILENAME_SST
+			    " not fully copied\n"
+			);
+		}
+		fclose(fp_out);
+		fclose(fp_in);
 	}
 
 	wsrep_uuid_t uuid;
@@ -99,7 +126,6 @@ xb_write_galera_info(bool incremental_prepare)
 		    "could not create " MB_GALERA_INFO_FILENAME
 		    ", errno = %d\n",
 		    errno);
-		exit(EXIT_FAILURE);
 	}
 
 	seqno = wsrep_xid_seqno(&xid);

extra/mariabackup/xb_plugin.h

@@ -1,5 +0,0 @@
-#include <mysql.h>
-#include <string>
-extern void xb_plugin_backup_init(MYSQL *mysql);
-extern const char* xb_plugin_get_config();
-extern void xb_plugin_prepare_init(int argc, char **argv, const char *dir);

extra/mariabackup/xbstream.cc

@@ -255,7 +255,7 @@ mode_create(int argc, char **argv)
 		return 1;
 	}
 
-	stream = xb_stream_write_new();
+	stream = xb_stream_write_new(nullptr, nullptr);
 	if (stream == NULL) {
 		msg("%s: xb_stream_write_new() failed.", my_progname);
 		return 1;
@@ -280,7 +280,7 @@ mode_create(int argc, char **argv)
 			goto err;
 		}
 
-		file = xb_stream_write_open(stream, filepath, &mystat, NULL, NULL);
+		file = xb_stream_write_open(stream, filepath, &mystat, false);
 		if (file == NULL) {
 			goto err;
 		}
@@ -307,7 +307,8 @@ err:
 
 static
 file_entry_t *
-file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
+file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen,
+	uchar chunk_flags)
 {
 	file_entry_t	*entry;
 	ds_file_t	*file;
@@ -324,7 +325,8 @@ file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
 	}
 	entry->pathlen = pathlen;
 
-	file = ds_open(ctxt->ds_ctxt, path, NULL);
+	file = ds_open(ctxt->ds_ctxt, path, NULL,
+		chunk_flags == XB_STREAM_FLAG_REWRITE);
 
 	if (file == NULL) {
 		msg("%s: failed to create file.", my_progname);
@@ -405,10 +407,50 @@ extract_worker_thread_func(void *arg)
 							(uchar *) chunk.path,
 							chunk.pathlen);
 
+		if (entry && (chunk.type == XB_CHUNK_TYPE_REMOVE ||
+			chunk.type == XB_CHUNK_TYPE_RENAME)) {
+			msg("%s: rename and remove chunks can not be applied to opened file: %s",
+				my_progname, chunk.path);
+			pthread_mutex_unlock(ctxt->mutex);
+			break;
+		}
+
+		if (chunk.type == XB_CHUNK_TYPE_REMOVE) {
+			if (ds_remove(ctxt->ds_ctxt, chunk.path)) {
+				msg("%s: error on file removing: %s", my_progname, chunk.path);
+				pthread_mutex_unlock(ctxt->mutex);
+				res = XB_STREAM_READ_ERROR;
+				break;
+			}
+			pthread_mutex_unlock(ctxt->mutex);
+			continue;
+		}
+
+		if (chunk.type == XB_CHUNK_TYPE_RENAME) {
+			if (my_hash_search(ctxt->filehash,
+				reinterpret_cast<const uchar *>(chunk.data), chunk.length)) {
+				msg("%s: rename chunks can not be applied to opened file: %s",
+					my_progname, reinterpret_cast<const uchar *>(chunk.data));
+				pthread_mutex_unlock(ctxt->mutex);
+				break;
+			}
+			if (ds_rename(ctxt->ds_ctxt, chunk.path,
+				reinterpret_cast<const char *>(chunk.data))) {
+				msg("%s: error on file renaming: %s to %s", my_progname,
+					reinterpret_cast<const char *>(chunk.data), chunk.path);
+				pthread_mutex_unlock(ctxt->mutex);
+				res = XB_STREAM_READ_ERROR;
+				break;
+			}
+			pthread_mutex_unlock(ctxt->mutex);
+			continue;
+		}
+
 		if (entry == NULL) {
 			entry = file_entry_new(ctxt,
 					       chunk.path,
-					       chunk.pathlen);
+					       chunk.pathlen,
+								 chunk.flags);
 			if (entry == NULL) {
 				pthread_mutex_unlock(ctxt->mutex);
 				break;
@@ -425,6 +467,18 @@ extract_worker_thread_func(void *arg)
 
 		pthread_mutex_unlock(ctxt->mutex);
 
+		if (chunk.type == XB_CHUNK_TYPE_SEEK) {
+			if (ds_seek_set(entry->file, chunk.offset)) {
+				msg("%s: my_seek() failed.", my_progname);
+				pthread_mutex_unlock(&entry->mutex);
+				res = XB_STREAM_READ_ERROR;
+				break;
+			}
+			entry->offset = chunk.offset;
+			pthread_mutex_unlock(&entry->mutex);
+			continue;
+		}
+
 		res = xb_stream_validate_checksum(&chunk);
 
 		if (res != XB_STREAM_READ_CHUNK) {

extra/mariabackup/xbstream.h

@@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 /* Chunk flags */
 /* Chunk can be ignored if unknown version/format */
 #define XB_STREAM_FLAG_IGNORABLE 0x01
+#define XB_STREAM_FLAG_REWRITE   0x02
 
 /* Magic + flags + type + path len */
 #define CHUNK_HEADER_CONSTANT_LEN ((sizeof(XB_STREAM_CHUNK_MAGIC) - 1) + \
@@ -48,18 +49,21 @@ typedef enum {
 /************************************************************************
 Write interface. */
 
-typedef ssize_t xb_stream_write_callback(xb_wstream_file_t *file,
+typedef ssize_t xb_stream_write_callback(
 					 void *userdata,
 					 const void *buf, size_t len);
 
-xb_wstream_t *xb_stream_write_new(void);
+xb_wstream_t *xb_stream_write_new(
-
+	xb_stream_write_callback *write_callback, void *user_data);
 xb_wstream_file_t *xb_stream_write_open(xb_wstream_t *stream, const char *path,
-					MY_STAT *mystat, void *userdata,
+					const MY_STAT *mystat, bool rewrite);
-					xb_stream_write_callback *onwrite);
 
 int xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len);
-
+int xb_stream_write_seek_set(xb_wstream_file_t *file, my_off_t offset);
+int xb_stream_write_remove(xb_wstream_t *stream, const char *path);
+int
+xb_stream_write_rename(
+	xb_wstream_t *stream, const char *old_path, const char *new_path);
 int xb_stream_write_close(xb_wstream_file_t *file);
 
 int xb_stream_write_done(xb_wstream_t *stream);
@@ -76,6 +80,9 @@ typedef enum {
 typedef enum {
 	XB_CHUNK_TYPE_UNKNOWN = '\0',
 	XB_CHUNK_TYPE_PAYLOAD = 'P',
+	XB_CHUNK_TYPE_RENAME = 'R',
+	XB_CHUNK_TYPE_REMOVE = 'D',
+	XB_CHUNK_TYPE_SEEK = 'S',
 	XB_CHUNK_TYPE_EOF = 'E'
 } xb_chunk_type_t;
 

extra/mariabackup/xbstream_read.cc

@@ -59,6 +59,9 @@ validate_chunk_type(uchar code)
 {
 	switch ((xb_chunk_type_t) code) {
 	case XB_CHUNK_TYPE_PAYLOAD:
+	case XB_CHUNK_TYPE_RENAME:
+	case XB_CHUNK_TYPE_REMOVE:
+	case XB_CHUNK_TYPE_SEEK:
 	case XB_CHUNK_TYPE_EOF:
 		return (xb_chunk_type_t) code;
 	default:
@@ -159,10 +162,29 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
 	}
 	chunk->path[pathlen] = '\0';
 
-	if (chunk->type == XB_CHUNK_TYPE_EOF) {
+	if (chunk->type == XB_CHUNK_TYPE_EOF ||
+			chunk->type == XB_CHUNK_TYPE_REMOVE) {
 		return XB_STREAM_READ_CHUNK;
 	}
 
+	if (chunk->type == XB_CHUNK_TYPE_RENAME) {
+		F_READ(tmpbuf, 4);
+		size_t new_pathlen = uint4korr(tmpbuf);
+		if (new_pathlen >= FN_REFLEN) {
+			msg("xb_stream_read_chunk(): path length (%lu) for new name of 'rename'"
+				"	chunk is too large", (ulong) new_pathlen);
+			goto err;
+		}
+		chunk->length = new_pathlen;
+		stream->offset +=4;
+	}
+	else if (chunk->type == XB_CHUNK_TYPE_SEEK) {
+		F_READ(tmpbuf, 8);
+		chunk->offset = uint8korr(tmpbuf);
+		stream->offset += 8;
+		return XB_STREAM_READ_CHUNK;
+	}
+	else {
 		/* Payload length */
 		F_READ(tmpbuf, 16);
 		ullval = uint8korr(tmpbuf);
@@ -185,19 +207,33 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
 		}
 		chunk->offset = (my_off_t) ullval;
 		stream->offset += 8;
+	}
 
-	/* Reallocate the buffer if needed */
+	/* Reallocate the buffer if needed, take into account trailing '\0' for
-	if (chunk->length > chunk->buflen) {
+	new file name in the case of XB_CHUNK_TYPE_RENAME */
-		chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data, chunk->length,
+	if (chunk->length + 1 > chunk->buflen) {
-					 MYF(MY_WME | MY_ALLOW_ZERO_PTR));
+		chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data,
+					chunk->length + 1, MYF(MY_WME | MY_ALLOW_ZERO_PTR));
 		if (chunk->data == NULL) {
 			msg("xb_stream_read_chunk(): failed to increase buffer "
-			    "to %lu bytes.", (ulong) chunk->length);
+			    "to %lu bytes.", (ulong) chunk->length + 1);
 			goto err;
 		}
-		chunk->buflen = chunk->length;
+		chunk->buflen = chunk->length + 1;
 	}
 
+	if (chunk->type == XB_CHUNK_TYPE_RENAME) {
+		if (chunk->length == 0) {
+			msg("xb_stream_read_chunk(): failed to read new name for file to rename "
+				": %s", chunk->path);
+			goto err;
+		}
+		F_READ(chunk->data, chunk->length);
+		stream->offset += chunk->length;
+		reinterpret_cast<char *>(chunk->data)[chunk->length] = '\0';
+		++chunk->length;
+	}
+	else {
 		/* Checksum */
 		F_READ(tmpbuf, 4);
 		chunk->checksum = uint4korr(tmpbuf);
@@ -210,6 +246,7 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
 		}
 
 		stream->offset += 4;
+	}
 
 	return XB_STREAM_READ_CHUNK;
 

extra/mariabackup/xbstream_write.cc

@@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 #include <my_global.h>
 #include <my_base.h>
 #include <zlib.h>
+#include <stdint.h>
 #include "common.h"
 #include "xbstream.h"
 
@@ -29,6 +30,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 
 struct xb_wstream_struct {
 	pthread_mutex_t	mutex;
+	xb_stream_write_callback *write;
+	void *user_data;
 };
 
 struct xb_wstream_file_struct {
@@ -39,8 +42,7 @@ struct xb_wstream_file_struct {
 	char		*chunk_ptr;
 	size_t		chunk_free;
 	my_off_t	offset;
-	void		*userdata;
+  bool rewrite;
-	xb_stream_write_callback *write;
 };
 
 static int xb_stream_flush(xb_wstream_file_t *file);
@@ -50,7 +52,7 @@ static int xb_stream_write_eof(xb_wstream_file_t *file);
 
 static
 ssize_t
-xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)),
+xb_stream_default_write_callback(
 				 void *userdata __attribute__((unused)),
 				 const void *buf, size_t len)
 {
@@ -60,21 +62,31 @@ xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused))
 }
 
 xb_wstream_t *
-xb_stream_write_new(void)
+xb_stream_write_new(
+	xb_stream_write_callback *write_callback, void *user_data)
 {
 	xb_wstream_t	*stream;
 
 	stream = (xb_wstream_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(xb_wstream_t), MYF(MY_FAE));
 	pthread_mutex_init(&stream->mutex, NULL);
+	if (write_callback) {
+#ifdef _WIN32
+		setmode(fileno(stdout), _O_BINARY);
+#endif
+		stream->write = write_callback;
+		stream->user_data = user_data;
+	}
+	else {
+		stream->write = xb_stream_default_write_callback;
+		stream->user_data = user_data;
+	}
 
 	return stream;;
 }
 
 xb_wstream_file_t *
 xb_stream_write_open(xb_wstream_t *stream, const char *path,
-		     MY_STAT *mystat __attribute__((unused)),
+		const MY_STAT *mystat __attribute__((unused)), bool rewrite)
-		     void *userdata,
-		     xb_stream_write_callback *onwrite)
 {
 	xb_wstream_file_t	*file;
 	size_t			path_len;
@@ -109,16 +121,7 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path,
 	file->offset = 0;
 	file->chunk_ptr = file->chunk;
 	file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE;
-	if (onwrite) {
+	file->rewrite = rewrite;
-#ifdef _WIN32
-		setmode(fileno(stdout), _O_BINARY);
-#endif
-		file->userdata = userdata;
-		file->write = onwrite;
-	} else {
-		file->userdata = NULL;
-		file->write = xb_stream_default_write_callback;
-	}
 
 	return file;
 }
@@ -202,7 +205,8 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len)
 	memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1);
 	ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1;
 
-	*ptr++ = 0;                              /* Chunk flags */
+	*ptr++ =
+    file->rewrite ? XB_STREAM_FLAG_REWRITE : 0; /* Chunk flags */
 
 	*ptr++ = (uchar) XB_CHUNK_TYPE_PAYLOAD;  /* Chunk type */
 
@@ -227,11 +231,11 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len)
 
 	xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
 
-	if (file->write(file, file->userdata, tmpbuf, ptr-tmpbuf) == -1)
+	if (stream->write(stream->user_data, tmpbuf, ptr-tmpbuf) == -1)
 		goto err;
 
 
-	if (file->write(file, file->userdata, buf, len) == -1) /* Payload */
+	if (stream->write(stream->user_data, buf, len) == -1) /* Payload */
 		goto err;
 
 	file->offset+= len;
@@ -247,6 +251,38 @@ err:
 	return 1;
 }
 
+int xb_stream_write_seek_set(xb_wstream_file_t *file, my_off_t offset)
+{
+	/* Chunk magic + flags + chunk type + path_len + path + offset */
+	uchar		tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 +
+			       FN_REFLEN + 8];
+	int error = 0;
+	xb_wstream_t	*stream = file->stream;
+	uchar		*ptr = tmpbuf;
+	/* Chunk magic */
+	memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1);
+	ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1;
+	*ptr++ = 0;                              /* Chunk flags */
+	*ptr++ = (uchar) XB_CHUNK_TYPE_SEEK;   /* Chunk type */
+	int4store(ptr, file->path_len);          /* Path length */
+	ptr += 4;
+	memcpy(ptr, file->path, file->path_len); /* Path */
+	ptr += file->path_len;
+	int8store(ptr, static_cast<int64_t>(offset));  /* Offset */
+	ptr += 8;
+	if (xb_stream_flush(file))
+		return 1;
+	pthread_mutex_lock(&stream->mutex);
+	if (stream->write(stream->user_data, tmpbuf, ptr-tmpbuf) == -1)
+		error = 1;
+	if (!error)
+		file->offset = offset;
+	pthread_mutex_unlock(&stream->mutex);
+	if (xb_stream_flush(file))
+		return 1;
+	return error;
+}
+
 static
 int
 xb_stream_write_eof(xb_wstream_file_t *file)
@@ -278,7 +314,7 @@ xb_stream_write_eof(xb_wstream_file_t *file)
 
 	xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
 
-	if (file->write(file, file->userdata, tmpbuf,
+	if (stream->write(stream->user_data, tmpbuf,
 			(ulonglong) (ptr - tmpbuf)) == -1)
 		goto err;
 
@@ -291,3 +327,77 @@ err:
 
 	return 1;
 }
+
+
+int
+xb_stream_write_remove(xb_wstream_t *stream, const char *path) {
+	/* Chunk magic + flags + chunk type + path_len + path */
+	uchar		tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + FN_REFLEN];
+	uchar		*ptr = tmpbuf;
+	/* Chunk magic */
+	memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1);
+	ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1;
+
+	*ptr++ = 0;                              /* Chunk flags */
+
+	*ptr++ = (uchar) XB_CHUNK_TYPE_REMOVE;   /* Chunk type */
+	size_t path_len = strlen(path);
+	int4store(ptr, path_len);               /* Path length */
+	ptr += 4;
+
+	memcpy(ptr, path, path_len);            /* Path */
+	ptr += path_len;
+
+	xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
+
+	pthread_mutex_lock(&stream->mutex);
+
+	ssize_t result = stream->write(stream->user_data, tmpbuf,
+		(ulonglong) (ptr - tmpbuf));
+
+	pthread_mutex_unlock(&stream->mutex);
+
+	return result < 0;
+
+}
+
+int
+xb_stream_write_rename(
+	xb_wstream_t *stream, const char *old_path, const char *new_path) {
+	/* Chunk magic + flags + chunk type + path_len + path + path_len + path*/
+	uchar		tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 +
+		4 + FN_REFLEN + 4 + FN_REFLEN];
+	uchar		*ptr = tmpbuf;
+	/* Chunk magic */
+	memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1);
+	ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1;
+
+	*ptr++ = 0;                              /* Chunk flags */
+
+	*ptr++ = (uchar) XB_CHUNK_TYPE_RENAME;   /* Chunk type */
+	size_t path_len = strlen(old_path);
+	int4store(ptr, path_len);               /* Path length */
+	ptr += 4;
+
+	memcpy(ptr, old_path, path_len);            /* Path */
+	ptr += path_len;
+
+	path_len = strlen(new_path);
+	int4store(ptr, path_len);               /* Path length */
+	ptr += 4;
+
+	memcpy(ptr, new_path, path_len);            /* Path */
+	ptr += path_len;
+
+	xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
+
+	pthread_mutex_lock(&stream->mutex);
+
+	ssize_t result = stream->write(stream->user_data, tmpbuf,
+		(ulonglong) (ptr - tmpbuf));
+
+	pthread_mutex_unlock(&stream->mutex);
+
+	return result < 0;
+}
+

extra/mariabackup/xtrabackup.h

@@ -24,8 +24,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 #include <my_getopt.h>
 #include "datasink.h"
 #include "xbstream.h"
-#include "changed_page_bitmap.h"
+#include "fil0fil.h"
 #include <set>
+#include "handler.h"
+
+#include <utility>
+#include <vector>
+#include <tuple>
+#include <functional>
+
 
 #define XB_TOOL_NAME "mariadb-backup"
 #define XB_HISTORY_TABLE "mysql.mariadb_backup_history"
@@ -84,8 +91,6 @@ extern my_bool	xb_backup_rocksdb;
 
 extern uint		opt_protocol;
 
-extern xb_page_bitmap *changed_page_bitmap;
-
 extern char		*xtrabackup_incremental;
 extern my_bool		xtrabackup_incremental_force_scan;
 
@@ -112,7 +117,7 @@ extern my_bool		xtrabackup_decrypt_decompress;
 extern char		*innobase_data_file_path;
 extern longlong		innobase_page_size;
 
-extern int		xtrabackup_parallel;
+extern uint		xtrabackup_parallel;
 
 extern my_bool		xb_close_files;
 extern const char	*xtrabackup_compress_alg;
@@ -131,7 +136,6 @@ extern my_bool		opt_galera_info;
 extern my_bool		opt_slave_info;
 extern my_bool		opt_no_lock;
 extern my_bool		opt_safe_slave_backup;
-extern my_bool		opt_rsync;
 extern my_bool		opt_force_non_empty_dirs;
 extern my_bool		opt_noversioncheck;
 extern my_bool		opt_no_backup_locks;
@@ -288,15 +292,40 @@ fil_file_readdir_next_file(
 	os_file_stat_t* info);	/*!< in/out: buffer where the
 				info is returned */
 
-#ifndef DBUG_OFF
+const char *convert_dst(const char *dst);
-#include <fil0fil.h>
-extern void dbug_mariabackup_event(const char *event,
-                            const fil_space_t::name_type key);
 
-#define DBUG_MARIABACKUP_EVENT(A, B)                                          \
+std::string get_table_version_from_image(const std::vector<uchar> &frm_image);
-  DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A, B);)
+std::pair<bool, legacy_db_type>
-#else
+	get_table_engine_from_image(const std::vector<uchar> &frm_image);
-#define DBUG_MARIABACKUP_EVENT(A, B) /* empty */
+std::string read_table_version_id(File file);
-#endif // DBUG_OFF
 
+std::string convert_tablename_to_filepath(
+	const char *data_dir_path, const std::string &db, const std::string &table);
+
+std::tuple<std::string, std::string, std::string>
+convert_filepath_to_tablename(const char *filepath);
+
+typedef std::string table_key_t;
+
+inline table_key_t table_key(const std::string &db, const std::string &table) {
+	return std::string(db).append(".").append(table);
+};
+
+inline table_key_t table_key(const char *db, const char *table) {
+	return std::string(db).append(".").append(table);
+};
+
+typedef std::function<void(std::string, std::string, std::string)>
+  post_copy_table_hook_t;
+
+my_bool
+check_if_skip_table(
+/******************/
+	const char*	name);	/*!< in: path to the table */
+
+bool is_log_table(const char *dbname, const char *tablename);
+bool is_stats_table(const char *dbname, const char *tablename);
+
+extern my_bool xtrabackup_copy_back;
+extern my_bool xtrabackup_move_back;
 #endif /* XB_XTRABACKUP_H */

extra/wolfssl/CMakeLists.txt

@@ -7,30 +7,21 @@ ELSE()
 ENDIF()
 
 IF(CMAKE_SIZEOF_VOID_P MATCHES 8)
-IF(MSVC_INTEL)
+IF(MSVC_INTEL AND NOT (CMAKE_C_COMPILER_ID MATCHES Clang))
   SET(WOLFSSL_INTELASM ON)
-  SET(WOLFSSL_X86_64_BUILD 1)
   SET(HAVE_INTEL_RDSEED 1)
   SET(HAVE_INTEL_RDRAND 1)
-ELSEIF(CMAKE_ASM_COMPILER_ID MATCHES "Clang" AND CMAKE_VERSION VERSION_LESS 3.16)
-
-  # WolfSSL 5.5.4 bug workaround below does not work, due to some CMake bug
 ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64")
-  SET(WOLFSSL_X86_64_BUILD 1)
   IF(CMAKE_C_COMPILER_ID MATCHES GNU  AND CMAKE_C_COMPILER_VERSION VERSION_LESS 4.9)
     MESSAGE_ONCE(NO_INTEL_ASSEMBLY "Disable Intel assembly for WolfSSL - compiler is too old")
-  ELSE()
+  ELSEIF(WITH_MSAN)
-    IF(WITH_MSAN)
+    MESSAGE_ONCE(MSAN_CANT_HANDLE_IT "Disable Intel assembly for WolfSSL - MSAN can't handle it")
-      MESSAGE_ONCE(MSAN_CANT_HANDLE_IT
-	"Disable Intel assembly for WolfSSL - MSAN can't handle it")
   ELSE()
     MY_CHECK_C_COMPILER_FLAG(-maes)
     MY_CHECK_C_COMPILER_FLAG(-msse4)
     MY_CHECK_C_COMPILER_FLAG(-mpclmul)
     IF(have_C__maes AND have_C__msse4 AND have_C__mpclmul)
       SET(WOLFSSL_INTELASM ON)
-      ENDIF()
-    ENDIF()
       MY_CHECK_C_COMPILER_FLAG(-mrdrnd)
       MY_CHECK_C_COMPILER_FLAG(-mrdseed)
       IF(have_C__mrdrnd)
@@ -42,11 +33,17 @@ ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64")
     ENDIF()
   ENDIF()
 ENDIF()
+ENDIF()
 
 SET(WOLFSSL_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/src)
+SET(WOLFCRYPT_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/wolfcrypt/src)
 ADD_DEFINITIONS(${SSL_DEFINES})
+ADD_DEFINITIONS(-DWOLFSSL_LIB -DBUILDING_WOLFSSL)
+ADD_DEFINITIONS(-DWOLFSSL_SP_4096)
+INCLUDE_DIRECTORIES(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl)
+INCLUDE_DIRECTORIES(${SSL_INCLUDE_DIRS})
 
-SET(WOLFSSL_SOURCES  
+add_library(wolfssl STATIC
 ${WOLFSSL_SRCDIR}/crl.c
 ${WOLFSSL_SRCDIR}/internal.c
 ${WOLFSSL_SRCDIR}/keys.c
@@ -54,33 +51,7 @@ SET(WOLFSSL_SOURCES
 ${WOLFSSL_SRCDIR}/wolfio.c
 ${WOLFSSL_SRCDIR}/ocsp.c
 ${WOLFSSL_SRCDIR}/ssl.c
-   ${WOLFSSL_SRCDIR}/tls13.c)
+${WOLFSSL_SRCDIR}/tls13.c
-
-ADD_DEFINITIONS(-DWOLFSSL_LIB -DBUILDING_WOLFSSL)
-
-INCLUDE_DIRECTORIES(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl)
-IF(MSVC)
-  # size_t to long truncation warning
-  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -wd4267 -wd4334 -wd4028 -wd4244")
-ENDIF()
-
-ADD_CONVENIENCE_LIBRARY(wolfssl ${WOLFSSL_SOURCES})
-
-# Workaround linker crash with older Ubuntu binutils
-# e.g aborting at ../../bfd/merge.c line 873 in _bfd_merged_section_offset
-IF(CMAKE_SYSTEM_NAME MATCHES "Linux")
-  STRING(REPLACE "-g " "-g1 " CMAKE_C_FLAGS_RELWITHDEBINFO 
-   ${CMAKE_C_FLAGS_RELWITHDEBINFO})
-  STRING(REPLACE "-g " "-g1 " CMAKE_C_FLAGS_DEBUG 
-     ${CMAKE_C_FLAGS_DEBUG})
-  STRING(REPLACE "-ggdb3 " " " CMAKE_C_FLAGS_RELWITHDEBINFO 
-    ${CMAKE_C_FLAGS_RELWITHDEBINFO})
-  STRING(REPLACE "-ggdb3 " " " CMAKE_C_FLAGS_DEBUG 
-     ${CMAKE_C_FLAGS_DEBUG})
-ENDIF()
-
-SET(WOLFCRYPT_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/wolfcrypt/src)
-SET(WOLFCRYPT_SOURCES
 ${WOLFCRYPT_SRCDIR}/aes.c
 ${WOLFCRYPT_SRCDIR}/arc4.c
 ${WOLFCRYPT_SRCDIR}/asn.c
@@ -110,69 +81,56 @@ ${WOLFCRYPT_SRCDIR}/wc_encrypt.c
 ${WOLFCRYPT_SRCDIR}/hash.c
 ${WOLFCRYPT_SRCDIR}/wolfmath.c
 ${WOLFCRYPT_SRCDIR}/kdf.c
+${WOLFCRYPT_SRCDIR}/sp_int.c
+${WOLFCRYPT_SRCDIR}/sp_c32.c
+${WOLFCRYPT_SRCDIR}/sp_c64.c
 )
 
-# Use fastmath large number math library.
+# Optimizations, assembly
-IF(NOT (MSVC AND CMAKE_C_COMPILER_ID MATCHES Clang))
+if(WOLFSSL_INTELASM)
-  # Can't use clang-cl with WOLFSSL_FASTMATH
+  set(WOLFSSL_X86_64_BUILD 1)
-  # due to https://bugs.llvm.org/show_bug.cgi?id=25305
+  set(WOLFSSL_SP_X86_64 1)
-  SET(WOLFSSL_FASTMATH 1)
+  set(WOLFSSL_SP_X86_64_ASM 1)
-ENDIF()
+  set(WOLFSSL_AESNI 1)
-
+  target_sources(wolfssl PRIVATE
-IF(WOLFSSL_FASTMATH)
+    ${WOLFCRYPT_SRCDIR}/cpuid.c
-  SET(USE_FAST_MATH 1)
+    ${WOLFCRYPT_SRCDIR}/sp_x86_64.c
-  SET(TFM_TIMING_RESISTANT 1)
+  )
-  # FP_MAX_BITS is set high solely to satisfy ssl_8k_key.test
+  if(MSVC_INTEL)
-  # WolfSSL will use more stack space with it
+    target_sources(wolfssl PRIVATE
-  SET(FP_MAX_BITS 16384)
-  SET(WOLFCRYPT_SOURCES  ${WOLFCRYPT_SOURCES} ${WOLFCRYPT_SRCDIR}/tfm.c)
-  IF((CMAKE_SIZEOF_VOID_P MATCHES 4) AND (CMAKE_SYSTEM_PROCESSOR MATCHES "86")
-     AND (NOT MSVC))
-   # Workaround https://github.com/wolfSSL/wolfssl/issues/4245
-   # On 32bit Intel, to satisfy inline assembly's wish for free registers
-   # 1. use -fomit-frame-pointer
-   # 2. With  GCC 4, additionally use -fno-PIC, which works on x86
-   #    (modern GCC has PIC optimizations, that make it unnecessary)
-   # The following assumes GCC or Clang
-   SET(TFM_COMPILE_FLAGS  "-fomit-frame-pointer")
-   IF(CMAKE_C_COMPILER_VERSION VERSION_LESS "5")
-     SET(TFM_COMPILE_FLAGS  "${TFM_COMPILE_FLAGS} -fno-PIC")
-   ENDIF()
-   SET_SOURCE_FILES_PROPERTIES(${WOLFCRYPT_SRCDIR}/tfm.c
-      PROPERTIES COMPILE_FLAGS ${TFM_COMPILE_FLAGS})
- ENDIF()
-ELSE()
-  SET(WOLFSSL_SP_MATH_ALL 1)
-  SET(WOLFCRYPT_SOURCES  ${WOLFCRYPT_SOURCES} ${WOLFCRYPT_SRCDIR}/sp_int.c)
-ENDIF()
-
-IF(WOLFSSL_X86_64_BUILD)
-  LIST(APPEND WOLFCRYPT_SOURCES ${WOLFCRYPT_SRCDIR}/cpuid.c)
-  IF(MSVC)
-    SET(WOLFSSL_AESNI 1)
-    LIST(APPEND WOLFCRYPT_SOURCES
       ${WOLFCRYPT_SRCDIR}/aes_asm.asm
-         ${WOLFCRYPT_SRCDIR}/aes_gcm_asm.asm)
+      ${WOLFCRYPT_SRCDIR}/aes_gcm_asm.asm
-    IF(CMAKE_C_COMPILER_ID MATCHES Clang)
+      ${WOLFCRYPT_SRCDIR}/sp_x86_64_asm.asm
-      SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes -msse4.2 -mpclmul -mrdrnd -mrdseed")
+    )
-    ENDIF()
+    target_compile_options(wolfssl PRIVATE
-  ELSEIF(WOLFSSL_INTELASM)
+      $<$<COMPILE_LANG_AND_ID:C,Clang>:-maes -msse4.2 -mpclmul -mrdrnd -mrdseed>
-    SET(WOLFSSL_AESNI 1)
+      $<$<COMPILE_LANGUAGE:ASM_MASM>:/Zi>
-    SET(USE_INTEL_SPEEDUP 1)
+    )
-    LIST(APPEND WOLFCRYPT_SOURCES
+  else()
+    set(USE_INTEL_SPEEDUP 1)
+    target_sources(wolfssl PRIVATE
       ${WOLFCRYPT_SRCDIR}/aes_asm.S
       ${WOLFCRYPT_SRCDIR}/aes_gcm_asm.S
       ${WOLFCRYPT_SRCDIR}/chacha_asm.S
       ${WOLFCRYPT_SRCDIR}/poly1305_asm.S
       ${WOLFCRYPT_SRCDIR}/sha512_asm.S
-      ${WOLFCRYPT_SRCDIR}/sha256_asm.S)
+      ${WOLFCRYPT_SRCDIR}/sha256_asm.S
-    ADD_DEFINITIONS(-maes -msse4.2 -mpclmul)
+      ${WOLFCRYPT_SRCDIR}/sp_x86_64_asm.S
-    # WolfSSL 5.5.4 bug - user_settings.h not included into aes_asm.S
+    )
-    SET_PROPERTY(SOURCE ${WOLFCRYPT_SRCDIR}/aes_asm.S APPEND PROPERTY COMPILE_OPTIONS "-DWOLFSSL_X86_64_BUILD")
+    target_compile_options(wolfssl PRIVATE -maes -msse4.2 -mpclmul)
-  ENDIF()
+    # Workaround 5.5.4 bug (user_settings.h not included into aes_asm.S)
-ENDIF()
+    set_property(SOURCE ${WOLFCRYPT_SRCDIR}/aes_asm.S APPEND PROPERTY COMPILE_OPTIONS "-DWOLFSSL_X86_64_BUILD")
+  endif()
+endif()
+
+# Silence some warnings
+if(MSVC)
+  # truncation warnings
+  target_compile_options(wolfssl PRIVATE $<$<COMPILE_LANGUAGE:C>:/wd4244>)
+  if(CMAKE_C_COMPILER_ID MATCHES Clang)
+    target_compile_options(wolfssl PRIVATE $<$<COMPILE_LANGUAGE:C>:-Wno-incompatible-function-pointer-types>)
+  endif()
+endif()
 
 CONFIGURE_FILE(user_settings.h.in user_settings.h)
-INCLUDE_DIRECTORIES(${SSL_INCLUDE_DIRS})
-ADD_CONVENIENCE_LIBRARY(wolfcrypt ${WOLFCRYPT_SOURCES})
 

extra/wolfssl/user_settings.h.in

@@ -21,6 +21,7 @@
 #define HAVE_AESGCM
 #define HAVE_CHACHA
 #define HAVE_POLY1305
+#define HAVE_THREAD_LS
 #define WOLFSSL_AES_COUNTER
 #define NO_WOLFSSL_STUB
 #define OPENSSL_ALL
@@ -53,20 +54,19 @@
 #define NO_RABBIT
 #define NO_RC4
 
-/*
-  FP_MAX_BITS is set high solely to satisfy ssl_8k_key.test
-  WolfSSL will use more stack space with it, with fastmath
-*/
-#cmakedefine FP_MAX_BITS 16384
 #define RSA_MAX_SIZE 8192
+#define WOLFSSL_SP_MATH_ALL
+#define WOLFSSL_HAVE_SP_RSA
+#ifndef WOLFSSL_SP_4096
+#define WOLFSSL_SP_4096
+#endif
+
 #cmakedefine WOLFSSL_AESNI
-#cmakedefine USE_FAST_MATH
-#cmakedefine TFM_TIMING_RESISTANT
 #cmakedefine HAVE_INTEL_RDSEED
 #cmakedefine HAVE_INTEL_RDRAND
 #cmakedefine USE_INTEL_SPEEDUP
-#cmakedefine USE_FAST_MATH
 #cmakedefine WOLFSSL_X86_64_BUILD
-#cmakedefine WOLFSSL_SP_MATH_ALL
+#cmakedefine WOLFSSL_SP_X86_64
+#cmakedefine WOLFSSL_SP_X86_64_ASM
 
 #endif  /* WOLFSSL_USER_SETTINGS_H */

extra/wolfssl/wolfssl

@@ -1 +1 @@
-Subproject commit 66596ad9e1d7efa8479656872cf09c9c1870a02e
+Subproject commit 8970ff4c34034dbb3594943d11f8c9d4c5512bd5

include/my_attribute.h

@@ -70,5 +70,19 @@
 # endif /* GNUC >= 3.1 */
 #endif
 
+/* Define pragmas to disable warnings for stack frame checking */
 
+#if defined(__clang__)
+#define PRAGMA_DISABLE_CHECK_STACK_FRAME                     \
+_Pragma("clang diagnostic push")                             \
+_Pragma("clang diagnostic ignored \"-Wframe-larger-than=\"")
+
+#define PRAGMA_REENABLE_CHECK_STACK_FRAME                    \
+_Pragma("clang diagnostic pop")
+
+#else
+#define PRAGMA_DISABLE_CHECK_STACK_FRAME
+#define PRAGMA_REENABLE_CHECK_STACK_FRAME
 #endif
+
+#endif /* _my_attribute_h */

include/my_base.h

@@ -49,6 +49,7 @@
 #define HA_OPEN_MERGE_TABLE		2048U
 #define HA_OPEN_FOR_CREATE              4096U
 #define HA_OPEN_FOR_DROP                (1U << 13) /* Open part of drop */
+#define HA_OPEN_GLOBAL_TMP_TABLE	(1U << 14) /* TMP table used by repliction */
 
 /*
   Allow opening even if table is incompatible as this is for ALTER TABLE which
@@ -377,6 +378,12 @@ enum ha_base_keytype {
 #define HA_CREATE_INTERNAL_TABLE 256U
 #define HA_PRESERVE_INSERT_ORDER 512U
 #define HA_CREATE_NO_ROLLBACK    1024U
+/*
+  A temporary table that can be used by different threads, eg. replication
+  threads. This flag ensure that memory is not allocated with THREAD_SPECIFIC,
+  as we do for other temporary tables.
+*/
+#define HA_CREATE_GLOBAL_TMP_TABLE 2048U
 
 /* Flags used by start_bulk_insert */
 

include/my_bitmap.h

@@ -22,14 +22,15 @@
 #include <m_string.h>
 #include <my_pthread.h>
 
-typedef uint32 my_bitmap_map;
+typedef ulonglong my_bitmap_map;
 
 typedef struct st_bitmap
 {
   my_bitmap_map *bitmap;
   my_bitmap_map *last_word_ptr;
-  my_bitmap_map last_word_mask;
+  my_bitmap_map last_bit_mask;
   uint32	n_bits; /* number of bits occupied by the above */
+  my_bool       bitmap_allocated;
 } MY_BITMAP;
 
 #ifdef	__cplusplus
@@ -39,7 +40,7 @@ extern "C" {
 /* Reset memory. Faster then doing a full bzero */
 #define my_bitmap_clear(A) ((A)->bitmap= 0)
 
-extern void create_last_word_mask(MY_BITMAP *map);
+extern void create_last_bit_mask(MY_BITMAP *map);
 extern my_bool my_bitmap_init(MY_BITMAP *map, my_bitmap_map *buf, uint n_bits);
 extern my_bool bitmap_is_clear_all(const MY_BITMAP *map);
 extern my_bool bitmap_is_prefix(const MY_BITMAP *map, uint prefix_size);
@@ -53,12 +54,12 @@ extern my_bool bitmap_fast_test_and_set(MY_BITMAP *map, uint bitmap_bit);
 extern my_bool bitmap_fast_test_and_clear(MY_BITMAP *map, uint bitmap_bit);
 extern my_bool bitmap_union_is_set_all(const MY_BITMAP *map1,
                                        const MY_BITMAP *map2);
-extern my_bool bitmap_exists_intersection(const MY_BITMAP **bitmap_array,
+extern my_bool bitmap_exists_intersection(MY_BITMAP **bitmap_array,
                                           uint bitmap_count,
                                           uint start_bit, uint end_bit);
 
 extern uint bitmap_set_next(MY_BITMAP *map);
-extern uint bitmap_get_first(const MY_BITMAP *map);
+extern uint bitmap_get_first_clear(const MY_BITMAP *map);
 extern uint bitmap_get_first_set(const MY_BITMAP *map);
 extern uint bitmap_bits_set(const MY_BITMAP *map);
 extern uint bitmap_get_next_set(const MY_BITMAP *map, uint bitmap_bit);
@@ -71,54 +72,70 @@ extern void bitmap_union(MY_BITMAP *map, const MY_BITMAP *map2);
 extern void bitmap_xor(MY_BITMAP *map, const MY_BITMAP *map2);
 extern void bitmap_invert(MY_BITMAP *map);
 extern void bitmap_copy(MY_BITMAP *map, const MY_BITMAP *map2);
+/* Functions to export/import bitmaps to an architecture independent format */
+extern void bitmap_export(uchar *to, MY_BITMAP *map);
+extern void bitmap_import(MY_BITMAP *map, uchar *from);
+
+#define my_bitmap_map_bytes sizeof(my_bitmap_map)
+#define my_bitmap_map_bits  (my_bitmap_map_bytes*8)
+/* Size in bytes to store 'bits' number of bits */
+#define bitmap_buffer_size(bits) (MY_ALIGN((bits), my_bitmap_map_bits)/8)
+#define my_bitmap_buffer_size(map) bitmap_buffer_size((map)->n_bits)
+#define no_bytes_in_export_map(map) (((map)->n_bits + 7)/8)
+#define no_words_in_map(map) (((map)->n_bits + (my_bitmap_map_bits-1))/my_bitmap_map_bits)
 
 /* Fast, not thread safe, bitmap functions */
-#define bitmap_buffer_size(bits) (((bits)+31)/32)*4
+/* The following functions must be compatible with create_last_bit_mask()! */
-#define no_bytes_in_map(map) (((map)->n_bits + 7)/8)
-#define no_words_in_map(map) (((map)->n_bits + 31)/32)
-#define bytes_word_aligned(bytes) (4*((bytes + 3)/4))
-/* The following functions must be compatible with create_last_word_mask()! */
 static inline void
 bitmap_set_bit(MY_BITMAP *map,uint bit)
 {
-  uchar *b= (uchar*) map->bitmap + bit / 8;
   DBUG_ASSERT(bit < map->n_bits);
-  *b= (uchar) (*b | 1U << (bit & 7));
+  map->bitmap[bit/my_bitmap_map_bits]|=
+    (1ULL << (bit & (my_bitmap_map_bits-1)));
 }
 static inline void
 bitmap_flip_bit(MY_BITMAP *map,uint bit)
 {
-  uchar *b= (uchar*) map->bitmap + bit / 8;
   DBUG_ASSERT(bit < map->n_bits);
-  *b= (uchar) (*b ^ 1U << (bit & 7));
+  map->bitmap[bit/my_bitmap_map_bits]^=
+    (1ULL << (bit & (my_bitmap_map_bits-1)));
 }
 static inline void
 bitmap_clear_bit(MY_BITMAP *map,uint bit)
 {
-  uchar *b= (uchar*) map->bitmap + bit / 8;
   DBUG_ASSERT(bit < map->n_bits);
-  *b= (uchar) (*b & ~(1U << (bit & 7)));
+  map->bitmap[bit/my_bitmap_map_bits]&=
+    ~(1ULL << (bit & (my_bitmap_map_bits-1)));
 }
 static inline uint
 bitmap_is_set(const MY_BITMAP *map,uint bit)
 {
-  const uchar *b= (const uchar*) map->bitmap + bit / 8;
   DBUG_ASSERT(bit < map->n_bits);
-  return !!(*b & (1U << (bit & 7)));
+  return (!!(map->bitmap[bit/my_bitmap_map_bits] &
+             (1ULL << (bit & (my_bitmap_map_bits-1)))));
 }
 
+/* Return true if bitmaps are equal */
 static inline my_bool bitmap_cmp(const MY_BITMAP *map1, const MY_BITMAP *map2)
 {
-  if (memcmp(map1->bitmap, map2->bitmap, 4*(no_words_in_map(map1)-1)) != 0)
+  DBUG_ASSERT(map1->n_bits == map2->n_bits);
-    return FALSE;
+  return (memcmp(map1->bitmap, map2->bitmap,
-  return ((*map1->last_word_ptr | map1->last_word_mask) ==
+                 my_bitmap_buffer_size(map1)) == 0);
-          (*map2->last_word_ptr | map2->last_word_mask));
 }
 
 #define bitmap_clear_all(MAP) \
-  { memset((MAP)->bitmap, 0, 4*no_words_in_map((MAP))); }
+  { memset((MAP)->bitmap, 0, my_bitmap_buffer_size(MAP)); }
-#define bitmap_set_all(MAP) \
+
-  (memset((MAP)->bitmap, 0xFF, 4*no_words_in_map((MAP))))
+static inline void
+bitmap_set_all(const MY_BITMAP *map)
+{
+  if (map->n_bits)
+  {
+    memset(map->bitmap, 0xFF, my_bitmap_map_bytes * (no_words_in_map(map)-1));
+    DBUG_ASSERT(map->bitmap + no_words_in_map(map)-1 == map->last_word_ptr);
+    *map->last_word_ptr= ~map->last_bit_mask;
+  }
+}
 
 #ifdef	__cplusplus
 }

include/my_global.h

@@ -974,6 +974,7 @@ typedef struct st_mysql_lex_string LEX_STRING;
 #define SOCKET_ECONNRESET WSAECONNRESET
 #define SOCKET_ENFILE	ENFILE
 #define SOCKET_EMFILE	EMFILE
+#define SOCKET_CLOSED   EIO
 #else /* Unix */
 #define socket_errno	errno
 #define closesocket(A)	close(A)
@@ -983,6 +984,7 @@ typedef struct st_mysql_lex_string LEX_STRING;
 #define SOCKET_EADDRINUSE EADDRINUSE
 #define SOCKET_ETIMEDOUT ETIMEDOUT
 #define SOCKET_ECONNRESET ECONNRESET
+#define SOCKET_CLOSED   EIO
 #define SOCKET_ENFILE	ENFILE
 #define SOCKET_EMFILE	EMFILE
 #endif

include/my_rdtsc.h

@@ -111,7 +111,7 @@ C_MODE_START
   On AARCH64, we use the generic timer base register. We override clang
   implementation for aarch64 as it access a PMU register which is not
   guaranteed to be active.
-  On RISC-V, we use the rdcycle instruction to read from mcycle register.
+  On RISC-V, we use the rdtime instruction to read from mtime register.
 
   Sadly, we have nothing for the Digital Alpha, MIPS, Motorola m68k,
   HP PA-RISC or other non-mainstream (or obsolete) processors.
@@ -211,15 +211,15 @@ static inline ulonglong my_timer_cycles(void)
   }
 #elif defined(__riscv)
   #define MY_TIMER_ROUTINE_CYCLES MY_TIMER_ROUTINE_RISCV
-  /* Use RDCYCLE (and RDCYCLEH on riscv32) */
+  /* Use RDTIME (and RDTIMEH on riscv32) */
   {
 # if __riscv_xlen == 32
     ulong result_lo, result_hi0, result_hi1;
     /* Implemented in assembly because Clang insisted on branching. */
     __asm __volatile__(
-        "rdcycleh %0\n"
+        "rdtimeh %0\n"
-        "rdcycle %1\n"
+        "rdtime %1\n"
-        "rdcycleh %2\n"
+        "rdtimeh %2\n"
         "sub %0, %0, %2\n"
         "seqz %0, %0\n"
         "sub %0, zero, %0\n"
@@ -228,7 +228,7 @@ static inline ulonglong my_timer_cycles(void)
     return (static_cast<ulonglong>(result_hi1) << 32) | result_lo;
 # else
     ulonglong result;
-    __asm __volatile__("rdcycle %0" : "=r"(result));
+    __asm __volatile__("rdtime %0" : "=r"(result));
     return result;
   }
 # endif

include/my_sys.h

@@ -154,7 +154,7 @@ char *guess_malloc_library();
 void sf_report_leaked_memory(my_thread_id id);
 int sf_sanity();
 extern my_thread_id (*sf_malloc_dbug_id)(void);
-#define SAFEMALLOC_REPORT_MEMORY(X) sf_report_leaked_memory(X)
+#define SAFEMALLOC_REPORT_MEMORY(X) if (!sf_leaking_memory) sf_report_leaked_memory(X)
 #else
 #define SAFEMALLOC_REPORT_MEMORY(X) do {} while(0)
 #endif
@@ -665,6 +665,7 @@ extern size_t my_fwrite(FILE *stream,const uchar *Buffer,size_t Count,
 		      myf MyFlags);
 extern my_off_t my_fseek(FILE *stream,my_off_t pos,int whence,myf MyFlags);
 extern my_off_t my_ftell(FILE *stream,myf MyFlags);
+extern void (*my_sleep_for_space)(unsigned int seconds);
 
 /* implemented in my_memmem.c */
 extern void *my_memmem(const void *haystack, size_t haystacklen,

include/mysql/service_print_check_msg.h

@@ -0,0 +1,44 @@
+/* Copyright (c) 2019, MariaDB Corporation.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA */
+
+#pragma once
+
+/**
+  @file include/mysql/service_print_check_msg.h
+  This service provides functions to write messages for check or repair
+*/
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct print_check_msg_service_st {
+  void (*print_check_msg)(MYSQL_THD, const char *db_name, const char *table_name,
+                          const char *op, const char *msg_type, const char *message,
+                          my_bool print_to_log);
+} *print_check_msg_service;
+
+#ifdef MYSQL_DYNAMIC_PLUGIN
+# define print_check_msg_context(_THD) print_check_msg_service->print_check_msg
+#else
+extern void print_check_msg(MYSQL_THD, const char *db_name, const char *table_name,
+                            const char *op, const char *msg_type, const char *message,
+                            my_bool print_to_log);
+#endif
+
+#ifdef __cplusplus
+}
+#endif

include/mysql_com.h

@@ -460,6 +460,7 @@ typedef struct st_net {
   my_bool thread_specific_malloc;
   unsigned char compress;
   my_bool pkt_nr_can_be_reset;
+  my_bool using_proxy_protocol;
   /*
     Pointer to query object in query cache, do not equal NULL (0) for
     queries in cache that have not stored its results yet

include/service_versions.h

@@ -44,6 +44,7 @@
 #define VERSION_wsrep                   0x0500
 #define VERSION_json                    0x0100
 #define VERSION_thd_mdl                 0x0100
+#define VERSION_print_check_msg         0x0100
 #define VERSION_sql_service             0x0102
 
 #define VERSION_provider_bzip2          0x0100

include/sslopt-longopts.h

@@ -19,13 +19,12 @@
 
 #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
 
-  {"ssl", OPT_SSL_SSL,
+  {"ssl", 0,
    "Enable SSL for connection (automatically enabled with other flags).",
    &opt_use_ssl, &opt_use_ssl, 0, GET_BOOL, OPT_ARG, 1, 0, 0, 0, 0, 0},
   {"ssl-ca", OPT_SSL_CA,
    "CA file in PEM format (check OpenSSL docs, implies --ssl).",
-   &opt_ssl_ca, &opt_ssl_ca, 0, GET_STR, REQUIRED_ARG,
+   &opt_ssl_ca, &opt_ssl_ca, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-   0, 0, 0, 0, 0, 0},
   {"ssl-capath", OPT_SSL_CAPATH,
    "CA directory (check OpenSSL docs, implies --ssl).",
    &opt_ssl_capath, &opt_ssl_capath, 0, GET_STR, REQUIRED_ARG,
@@ -57,7 +56,7 @@
   {"ssl-fplist", OPT_SSL_FPLIST, "File with accepted server certificate "
    "fingerprints, one per line (implies --ssl).",
    &opt_ssl_fplist, &opt_ssl_fplist, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"ssl-verify-server-cert", OPT_SSL_VERIFY_SERVER_CERT,
+  {"ssl-verify-server-cert", 0,
    "Verify server's certificate to prevent man-in-the-middle attacks",
    &opt_ssl_verify_server_cert, &opt_ssl_verify_server_cert,
    0, GET_BOOL, OPT_ARG, 2, 0, 0, 0, 0, 0},

include/violite.h

@@ -41,6 +41,13 @@ enum enum_vio_type
   VIO_TYPE_SSL
   /* see also vio_type_names[] */
 };
+
+enum enum_vio_state
+{
+  VIO_STATE_NOT_INITIALIZED, VIO_STATE_ACTIVE, VIO_STATE_SHUTDOWN,
+  VIO_STATE_CLOSED
+};
+
 #define FIRST_VIO_TYPE VIO_CLOSED
 #define LAST_VIO_TYPE VIO_TYPE_SSL
 
@@ -244,6 +251,7 @@ struct st_vio
   struct sockaddr_storage local;	/* Local internet address */
   struct sockaddr_storage remote;	/* Remote internet address */
   enum enum_vio_type	type;		/* Type of connection */
+  enum enum_vio_state	state;		/* State of the connection */
   const char		*desc;		/* String description */
   char                  *read_buffer;   /* buffer for vio_read_buff */
   char                  *read_pos;      /* start of unfetched data in the

libmariadb

@@ -1 +1 @@
-Subproject commit b4d75e78c487254cab9c42e259c3522e9cd0c7c4
+Subproject commit d9a50aceac6496215f8fdadc07658d923bb41afd

libmysqld/CMakeLists.txt

@@ -25,7 +25,7 @@ ${CMAKE_SOURCE_DIR}/tpool
 ${CMAKE_BINARY_DIR}/sql 
 ${PCRE_INCLUDE_DIRS}
 ${LIBFMT_INCLUDE_DIR}
-${ZLIB_INCLUDE_DIR}
+${ZLIB_INCLUDE_DIRS}
 ${SSL_INCLUDE_DIRS}
 ${SSL_INTERNAL_INCLUDE_DIRS}
 )
@@ -185,7 +185,7 @@ ENDIF()
 
 SET(LIBS 
   dbug strings mysys mysys_ssl pcre2-8 vio
-  ${ZLIB_LIBRARY} ${SSL_LIBRARIES} 
+  ${ZLIB_LIBRARIES} ${SSL_LIBRARIES}
   ${LIBWRAP} ${LIBCRYPT} ${CMAKE_DL_LIBS}
   ${EMBEDDED_PLUGIN_LIBS}
   sql_embedded

libmysqld/embedded_priv.h

@@ -23,6 +23,8 @@ void init_embedded_mysql(MYSQL *mysql, ulong client_flag);
 void *create_embedded_thd(ulong client_flag);
 int check_embedded_connection(MYSQL *mysql, const char *db);
 void free_old_query(MYSQL *mysql);
+THD *embedded_get_current_thd();
+void embedded_set_current_thd(THD *thd);
 extern MYSQL_METHODS embedded_methods;
 
 /* This one is used by embedded library to gather returning data */