1berry/MariaDB-server
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 10.2 into bb-10.2-ext

Browse Source
This commit is contained in:
Marko Mäkelä 2017-06-19 16:56:13 +03:00
commit 3a7201ea92
61 changed files with 286 additions and 4423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
extra/mariabackup/backup_copy.cc
View File

@ -1774,7 +1774,7 @@ copy_back()
const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile", const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile",
"xtrabackup_binary", "xtrabackup_binlog_info", "xtrabackup_binary", "xtrabackup_binlog_info",
"xtrabackup_checkpoints", ".qp", ".pmap", ".tmp", "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp",
".xbcrypt", NULL}; NULL};
const char *filename; const char *filename;
char c_tmp; char c_tmp;
int i_tmp; int i_tmp;
@ -1807,7 +1807,7 @@ copy_back()
filename = base_name(node.filepath); filename = base_name(node.filepath);
/* skip .qp and .xbcrypt files */ /* skip .qp files */
if (filename_matches(filename, ext_list)) { if (filename_matches(filename, ext_list)) {
continue; continue;
} }
@ -1899,24 +1899,8 @@ decrypt_decompress_file(const char *filepath, uint thread_n)
cmd << IF_WIN("type ","cat ") << filepath; cmd << IF_WIN("type ","cat ") << filepath;
if (ends_with(filepath, ".xbcrypt") && opt_decrypt) {
cmd << " | xbcrypt --decrypt --encrypt-algo="
<< xtrabackup_encrypt_algo_names[opt_decrypt_algo];
if (xtrabackup_encrypt_key) {
cmd << " --encrypt-key=" << xtrabackup_encrypt_key;
} else {
cmd << " --encrypt-key-file="
<< xtrabackup_encrypt_key_file;
}
dest_filepath[strlen(dest_filepath) - 8] = 0;
message << "decrypting";
needs_action = true;
}
if (opt_decompress if (opt_decompress
&& (ends_with(filepath, ".qp") && ends_with(filepath, ".qp")) {
|| (ends_with(filepath, ".qp.xbcrypt")
&& opt_decrypt))) {
cmd << " | qpress -dio "; cmd << " | qpress -dio ";
dest_filepath[strlen(dest_filepath) - 3] = 0; dest_filepath[strlen(dest_filepath) - 3] = 0;
if (needs_action) { if (needs_action) {
@ -1967,8 +1951,7 @@ decrypt_decompress_thread_func(void *arg)
continue; continue;
} }
if (!ends_with(node.filepath, ".qp") if (!ends_with(node.filepath, ".qp")) {
&& !ends_with(node.filepath, ".xbcrypt")) {
continue; continue;
} }

28
extra/mariabackup/backup_mysql.cc
View File

@ -1435,9 +1435,7 @@ write_xtrabackup_info(MYSQL *connection)
"partial = %s\n" "partial = %s\n"
"incremental = %s\n" "incremental = %s\n"
"format = %s\n" "format = %s\n"
"compact = %s\n" "compressed = %s\n",
"compressed = %s\n"
"encrypted = %s\n",
uuid, /* uuid */ uuid, /* uuid */
opt_history ? opt_history : "", /* name */ opt_history ? opt_history : "", /* name */
tool_name, /* tool_name */ tool_name, /* tool_name */
@ -1455,9 +1453,7 @@ write_xtrabackup_info(MYSQL *connection)
is_partial? "Y" : "N", is_partial? "Y" : "N",
xtrabackup_incremental ? "Y" : "N", /* incremental */ xtrabackup_incremental ? "Y" : "N", /* incremental */
xb_stream_name[xtrabackup_stream_fmt], /* format */ xb_stream_name[xtrabackup_stream_fmt], /* format */
"N", /* compact */ xtrabackup_compress ? "compressed" : "N"); /* compressed */
xtrabackup_compress ? "compressed" : "N", /* compressed */
xtrabackup_encrypt ? "Y" : "N"); /* encrypted */
if (!opt_history) { if (!opt_history) {
goto cleanup; goto cleanup;
@ -1483,9 +1479,7 @@ write_xtrabackup_info(MYSQL *connection)
"partial ENUM('Y', 'N') DEFAULT NULL," "partial ENUM('Y', 'N') DEFAULT NULL,"
"incremental ENUM('Y', 'N') DEFAULT NULL," "incremental ENUM('Y', 'N') DEFAULT NULL,"
"format ENUM('file', 'tar', 'xbstream') DEFAULT NULL," "format ENUM('file', 'tar', 'xbstream') DEFAULT NULL,"
"compact ENUM('Y', 'N') DEFAULT NULL," "compressed ENUM('Y', 'N') DEFAULT NULL"
"compressed ENUM('Y', 'N') DEFAULT NULL,"
"encrypted ENUM('Y', 'N') DEFAULT NULL"
") CHARACTER SET utf8 ENGINE=innodb", false); ") CHARACTER SET utf8 ENGINE=innodb", false);
@ -1495,8 +1489,8 @@ write_xtrabackup_info(MYSQL *connection)
<< "uuid, name, tool_name, tool_command, tool_version," << "uuid, name, tool_name, tool_command, tool_version,"
<< "ibbackup_version, server_version, start_time, end_time," << "ibbackup_version, server_version, start_time, end_time,"
<< "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn," << "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn,"
<< "partial, incremental, format, compact, compressed, " << "partial, incremental, format, compressed) "
<< "encrypted) values(" << "values("
<< escape_and_quote(connection, uuid) << "," << escape_and_quote(connection, uuid) << ","
<< escape_and_quote(connection, opt_history) << "," << escape_and_quote(connection, opt_history) << ","
<< escape_and_quote(connection, tool_name) << "," << escape_and_quote(connection, tool_name) << ","
@ -1513,9 +1507,7 @@ write_xtrabackup_info(MYSQL *connection)
<< ESCAPE_BOOL(is_partial) << "," << ESCAPE_BOOL(is_partial) << ","
<< ESCAPE_BOOL(xtrabackup_incremental)<< "," << ESCAPE_BOOL(xtrabackup_incremental)<< ","
<< escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<"," << escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<","
<< ESCAPE_BOOL(false) << "," << ESCAPE_BOOL(xtrabackup_compress) << ")";
<< ESCAPE_BOOL(xtrabackup_compress) << ","
<< ESCAPE_BOOL(xtrabackup_encrypt) <<")";
xb_mysql_query(mysql_connection, oss.str().c_str(), false); xb_mysql_query(mysql_connection, oss.str().c_str(), false);
@ -1581,14 +1573,6 @@ char *make_argv(char *buf, size_t len, int argc, char **argv)
if (strncmp(*argv, "--password", strlen("--password")) == 0) { if (strncmp(*argv, "--password", strlen("--password")) == 0) {
arg = "--password=..."; arg = "--password=...";
} }
if (strncmp(*argv, "--encrypt-key",
strlen("--encrypt-key")) == 0) {
arg = "--encrypt-key=...";
}
if (strncmp(*argv, "--encrypt_key",
strlen("--encrypt_key")) == 0) {
arg = "--encrypt_key=...";
}
left-= ut_snprintf(buf + len - left, left, left-= ut_snprintf(buf + len - left, left,
"%s%c", arg, argc > 1 ? ' ' : 0); "%s%c", arg, argc > 1 ? ' ' : 0);
++argv; --argc; ++argv; --argc;

665
extra/mariabackup/ds_decrypt.c
View File

@ -1,665 +0,0 @@
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption datasink implementation for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#include "common.h"
#include "datasink.h"
#include "xbcrypt.h"
#include "xbcrypt_common.h"
#include "crc_glue.h"
typedef struct {
pthread_t id;
uint num;
pthread_mutex_t ctrl_mutex;
pthread_cond_t ctrl_cond;
pthread_mutex_t data_mutex;
pthread_cond_t data_cond;
my_bool started;
my_bool data_avail;
my_bool cancelled;
my_bool failed;
const uchar *from;
size_t from_len;
uchar *to;
size_t to_len;
size_t to_size;
const uchar *iv;
size_t iv_len;
unsigned long long offset;
my_bool hash_appended;
gcry_cipher_hd_t cipher_handle;
xb_rcrypt_result_t parse_result;
} crypt_thread_ctxt_t;
typedef struct {
crypt_thread_ctxt_t *threads;
uint nthreads;
int encrypt_algo;
size_t chunk_size;
char *encrypt_key;
char *encrypt_key_file;
} ds_decrypt_ctxt_t;
typedef struct {
ds_decrypt_ctxt_t *crypt_ctxt;
size_t bytes_processed;
ds_file_t *dest_file;
uchar *buf;
size_t buf_len;
size_t buf_size;
} ds_decrypt_file_t;
int ds_decrypt_encrypt_threads = 1;
static ds_ctxt_t *decrypt_init(const char *root);
static ds_file_t *decrypt_open(ds_ctxt_t *ctxt, const char *path,
MY_STAT *mystat);
static int decrypt_write(ds_file_t *file, const void *buf, size_t len);
static int decrypt_close(ds_file_t *file);
static void decrypt_deinit(ds_ctxt_t *ctxt);
datasink_t datasink_decrypt = {
&decrypt_init,
&decrypt_open,
&decrypt_write,
&decrypt_close,
&decrypt_deinit
};
static crypt_thread_ctxt_t *create_worker_threads(uint n);
static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
static void *decrypt_worker_thread_func(void *arg);
static
ds_ctxt_t *
decrypt_init(const char *root)
{
ds_ctxt_t *ctxt;
ds_decrypt_ctxt_t *decrypt_ctxt;
crypt_thread_ctxt_t *threads;
if (xb_crypt_init(NULL)) {
return NULL;
}
/* Create and initialize the worker threads */
threads = create_worker_threads(ds_decrypt_encrypt_threads);
if (threads == NULL) {
msg("decrypt: failed to create worker threads.\n");
return NULL;
}
ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +
sizeof(ds_decrypt_ctxt_t),
MYF(MY_FAE));
decrypt_ctxt = (ds_decrypt_ctxt_t *) (ctxt + 1);
decrypt_ctxt->threads = threads;
decrypt_ctxt->nthreads = ds_decrypt_encrypt_threads;
ctxt->ptr = decrypt_ctxt;
ctxt->root = my_strdup(root, MYF(MY_FAE));
return ctxt;
}
static
ds_file_t *
decrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
{
ds_ctxt_t *dest_ctxt;
ds_decrypt_ctxt_t *crypt_ctxt;
ds_decrypt_file_t *crypt_file;
char new_name[FN_REFLEN];
ds_file_t *file;
xb_ad(ctxt->pipe_ctxt != NULL);
dest_ctxt = ctxt->pipe_ctxt;
crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr;
file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
sizeof(ds_decrypt_file_t),
MYF(MY_FAE|MY_ZEROFILL));
crypt_file = (ds_decrypt_file_t *) (file + 1);
/* Remove the .xbcrypt extension from the filename */
strncpy(new_name, path, FN_REFLEN);
new_name[strlen(new_name) - 8] = 0;
crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat);
if (crypt_file->dest_file == NULL) {
msg("decrypt: ds_open(\"%s\") failed.\n", new_name);
goto err;
}
crypt_file->crypt_ctxt = crypt_ctxt;
crypt_file->buf = NULL;
crypt_file->buf_size = 0;
crypt_file->buf_len = 0;
file->ptr = crypt_file;
file->path = crypt_file->dest_file->path;
return file;
err:
if (crypt_file->dest_file) {
ds_close(crypt_file->dest_file);
}
my_free(file);
return NULL;
}
#define CHECK_BUF_SIZE(ptr, size, buf, len) \
if (ptr + size - buf > (ssize_t) len) { \
result = XB_CRYPT_READ_INCOMPLETE; \
goto exit; \
}
static
xb_rcrypt_result_t
parse_xbcrypt_chunk(crypt_thread_ctxt_t *thd, const uchar *buf, size_t len,
size_t *bytes_processed)
{
const uchar *ptr;
uint version;
ulong checksum, checksum_exp;
ulonglong tmp;
xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK;
*bytes_processed = 0;
ptr = buf;
CHECK_BUF_SIZE(ptr, XB_CRYPT_CHUNK_MAGIC_SIZE, buf, len);
if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 3;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 2;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 1;
} else {
msg("%s:%s: wrong chunk magic at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
thd->offset += XB_CRYPT_CHUNK_MAGIC_SIZE;
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* reserved */
ptr += 8;
thd->offset += 8;
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* original size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid original size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += 8;
thd->to_len = (size_t)tmp;
if (thd->to_size < thd->to_len + XB_CRYPT_HASH_LEN) {
thd->to = (uchar *) my_realloc(
thd->to,
thd->to_len + XB_CRYPT_HASH_LEN,
MYF(MY_FAE | MY_ALLOW_ZERO_PTR));
thd->to_size = thd->to_len;
}
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr); /* encrypted size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid encrypted size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += 8;
thd->from_len = (size_t)tmp;
xb_a(thd->from_len <= thd->to_len + XB_CRYPT_HASH_LEN);
CHECK_BUF_SIZE(ptr, 4, buf, len);
checksum_exp = uint4korr(ptr); /* checksum */
ptr += 4;
thd->offset += 4;
/* iv size */
if (version == 1) {
thd->iv_len = 0;
thd->iv = NULL;
} else {
CHECK_BUF_SIZE(ptr, 8, buf, len);
tmp = uint8korr(ptr);
if (tmp > INT_MAX) {
msg("%s:%s: invalid iv size at offset 0x%llx.\n",
my_progname, __FUNCTION__, thd->offset);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
ptr += 8;
thd->offset += 8;
thd->iv_len = (size_t)tmp;
}
if (thd->iv_len > 0) {
CHECK_BUF_SIZE(ptr, thd->iv_len, buf, len);
thd->iv = ptr;
ptr += thd->iv_len;
}
/* for version euqals 2 we need to read in the iv data but do not init
CTR with it */
if (version == 2) {
thd->iv_len = 0;
thd->iv = 0;
}
if (thd->from_len > 0) {
CHECK_BUF_SIZE(ptr, thd->from_len, buf, len);
thd->from = ptr;
ptr += thd->from_len;
}
xb_ad(thd->from_len <= thd->to_len);
checksum = crc32_iso3309(0, thd->from, thd->from_len);
if (checksum != checksum_exp) {
msg("%s:%s invalid checksum at offset 0x%llx, "
"expected 0x%lx, actual 0x%lx.\n", my_progname,
__FUNCTION__, thd->offset, checksum_exp, checksum);
result = XB_CRYPT_READ_ERROR;
goto exit;
}
thd->offset += thd->from_len;
thd->hash_appended = version > 2;
exit:
*bytes_processed = (size_t) (ptr - buf);
return result;
}
static
int
decrypt_write(ds_file_t *file, const void *buf, size_t len)
{
ds_decrypt_file_t *crypt_file;
ds_decrypt_ctxt_t *crypt_ctxt;
crypt_thread_ctxt_t *threads;
crypt_thread_ctxt_t *thd;
uint nthreads;
uint i;
size_t bytes_processed;
xb_rcrypt_result_t parse_result = XB_CRYPT_READ_CHUNK;
my_bool err = FALSE;
crypt_file = (ds_decrypt_file_t *) file->ptr;
crypt_ctxt = crypt_file->crypt_ctxt;
threads = crypt_ctxt->threads;
nthreads = crypt_ctxt->nthreads;
if (crypt_file->buf_len > 0) {
thd = threads;
pthread_mutex_lock(&thd->ctrl_mutex);
do {
if (parse_result == XB_CRYPT_READ_INCOMPLETE) {
crypt_file->buf_size = crypt_file->buf_size * 2;
crypt_file->buf = (uchar *) my_realloc(
crypt_file->buf,
crypt_file->buf_size,
MYF(MY_FAE|MY_ALLOW_ZERO_PTR));
}
memcpy(crypt_file->buf + crypt_file->buf_len,
buf, MY_MIN(crypt_file->buf_size -
crypt_file->buf_len, len));
parse_result = parse_xbcrypt_chunk(
thd, crypt_file->buf,
crypt_file->buf_size, &bytes_processed);
if (parse_result == XB_CRYPT_READ_ERROR) {
pthread_mutex_unlock(&thd->ctrl_mutex);
return 1;
}
} while (parse_result == XB_CRYPT_READ_INCOMPLETE &&
crypt_file->buf_size < len);
if (parse_result != XB_CRYPT_READ_CHUNK) {
msg("decrypt: incomplete data.\n");
pthread_mutex_unlock(&thd->ctrl_mutex);
return 1;
}
pthread_mutex_lock(&thd->data_mutex);
thd->data_avail = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
len -= bytes_processed - crypt_file->buf_len;
buf += bytes_processed - crypt_file->buf_len;
/* reap */
pthread_mutex_lock(&thd->data_mutex);
while (thd->data_avail == TRUE) {
pthread_cond_wait(&thd->data_cond,
&thd->data_mutex);
}
if (thd->failed) {
msg("decrypt: failed to decrypt chunk.\n");
err = TRUE;
}
xb_a(thd->to_len > 0);
if (!err &&
ds_write(crypt_file->dest_file, thd->to, thd->to_len)) {
msg("decrypt: write to destination failed.\n");
err = TRUE;
}
crypt_file->bytes_processed += thd->from_len;
pthread_mutex_unlock(&thd->data_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
crypt_file->buf_len = 0;
if (err) {
return 1;
}
}
while (parse_result == XB_CRYPT_READ_CHUNK && len > 0) {
uint max_thread;
for (i = 0; i < nthreads; i++) {
thd = threads + i;
pthread_mutex_lock(&thd->ctrl_mutex);
parse_result = parse_xbcrypt_chunk(
thd, buf, len, &bytes_processed);
if (parse_result == XB_CRYPT_READ_ERROR) {
pthread_mutex_unlock(&thd->ctrl_mutex);
err = TRUE;
break;
}
thd->parse_result = parse_result;
if (parse_result != XB_CRYPT_READ_CHUNK) {
pthread_mutex_unlock(&thd->ctrl_mutex);
break;
}
pthread_mutex_lock(&thd->data_mutex);
thd->data_avail = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
len -= bytes_processed;
buf += bytes_processed;
}
max_thread = (i < nthreads) ? i : nthreads - 1;
/* Reap and write decrypted data */
for (i = 0; i <= max_thread; i++) {
thd = threads + i;
if (thd->parse_result != XB_CRYPT_READ_CHUNK) {
break;
}
pthread_mutex_lock(&thd->data_mutex);
while (thd->data_avail == TRUE) {
pthread_cond_wait(&thd->data_cond,
&thd->data_mutex);
}
if (thd->failed) {
msg("decrypt: failed to decrypt chunk.\n");
err = TRUE;
}
xb_a(thd->to_len > 0);
if (!err && ds_write(crypt_file->dest_file, thd->to,
thd->to_len)) {
msg("decrypt: write to destination failed.\n");
err = TRUE;
}
crypt_file->bytes_processed += thd->from_len;
pthread_mutex_unlock(&thd->data_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
}
if (err) {
return 1;
}
}
if (parse_result == XB_CRYPT_READ_INCOMPLETE && len > 0) {
crypt_file->buf_len = len;
if (crypt_file->buf_size < len) {
crypt_file->buf = (uchar *) my_realloc(
crypt_file->buf,
crypt_file->buf_len,
MYF(MY_FAE | MY_ALLOW_ZERO_PTR));
crypt_file->buf_size = len;
}
memcpy(crypt_file->buf, buf, len);
}
return 0;
}
static
int
decrypt_close(ds_file_t *file)
{
ds_decrypt_file_t *crypt_file;
ds_file_t *dest_file;
int rc = 0;
crypt_file = (ds_decrypt_file_t *) file->ptr;
dest_file = crypt_file->dest_file;
if (ds_close(dest_file)) {
rc = 1;
}
my_free(crypt_file->buf);
my_free(file);
return rc;
}
static
void
decrypt_deinit(ds_ctxt_t *ctxt)
{
ds_decrypt_ctxt_t *crypt_ctxt;
xb_ad(ctxt->pipe_ctxt != NULL);
crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr;
destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads);
my_free(ctxt->root);
my_free(ctxt);
}
static
crypt_thread_ctxt_t *
create_worker_threads(uint n)
{
crypt_thread_ctxt_t *threads;
uint i;
threads = (crypt_thread_ctxt_t *)
my_malloc(sizeof(crypt_thread_ctxt_t) * n,
MYF(MY_FAE | MY_ZEROFILL));
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
thd->num = i + 1;
/* Initialize the control mutex and condition var */
if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
pthread_cond_init(&thd->ctrl_cond, NULL)) {
goto err;
}
/* Initialize and data mutex and condition var */
if (pthread_mutex_init(&thd->data_mutex, NULL) ||
pthread_cond_init(&thd->data_cond, NULL)) {
goto err;
}
xb_crypt_cipher_open(&thd->cipher_handle);
pthread_mutex_lock(&thd->ctrl_mutex);
if (pthread_create(&thd->id, NULL, decrypt_worker_thread_func,
thd)) {
msg("decrypt: pthread_create() failed: "
"errno = %d\n", errno);
goto err;
}
}
/* Wait for the threads to start */
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
while (thd->started == FALSE)
pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
}
return threads;
err:
return NULL;
}
static
void
destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
{
uint i;
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
pthread_mutex_lock(&thd->data_mutex);
threads[i].cancelled = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
pthread_join(thd->id, NULL);
pthread_cond_destroy(&thd->data_cond);
pthread_mutex_destroy(&thd->data_mutex);
pthread_cond_destroy(&thd->ctrl_cond);
pthread_mutex_destroy(&thd->ctrl_mutex);
xb_crypt_cipher_close(thd->cipher_handle);
my_free(thd->to);
}
my_free(threads);
}
static
void *
decrypt_worker_thread_func(void *arg)
{
crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg;
pthread_mutex_lock(&thd->ctrl_mutex);
pthread_mutex_lock(&thd->data_mutex);
thd->started = TRUE;
pthread_cond_signal(&thd->ctrl_cond);
pthread_mutex_unlock(&thd->ctrl_mutex);
while (1) {
thd->data_avail = FALSE;
pthread_cond_signal(&thd->data_cond);
while (!thd->data_avail && !thd->cancelled) {
pthread_cond_wait(&thd->data_cond, &thd->data_mutex);
}
if (thd->cancelled)
break;
if (xb_crypt_decrypt(thd->cipher_handle, thd->from,
thd->from_len, thd->to, &thd->to_len,
thd->iv, thd->iv_len,
thd->hash_appended)) {
thd->failed = TRUE;
continue;
}
}
pthread_mutex_unlock(&thd->data_mutex);
return NULL;
}

30
extra/mariabackup/ds_decrypt.h
View File

@ -1,30 +0,0 @@
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef DS_DECRYPT_H
#define DS_DECRYPT_H
#include "datasink.h"
extern datasink_t datasink_decrypt;
extern int ds_decrypt_encrypt_threads;
#endif

446
extra/mariabackup/ds_encrypt.c
View File

@ -1,446 +0,0 @@
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
Encryption datasink implementation for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#include "common.h"
#include "datasink.h"
#include "xbcrypt_common.h"
#ifdef HAVE_GRYPT
#include "xbcrypt.h"
#define XB_CRYPT_CHUNK_SIZE ((size_t) (ds_encrypt_encrypt_chunk_size))
typedef struct {
pthread_t id;
uint num;
pthread_mutex_t ctrl_mutex;
pthread_cond_t ctrl_cond;
pthread_mutex_t data_mutex;
pthread_cond_t data_cond;
my_bool started;
my_bool data_avail;
my_bool cancelled;
const uchar *from;
size_t from_len;
uchar *to;
uchar *iv;
size_t to_len;
gcry_cipher_hd_t cipher_handle;
} crypt_thread_ctxt_t;
typedef struct {
crypt_thread_ctxt_t *threads;
uint nthreads;
} ds_encrypt_ctxt_t;
typedef struct {
xb_wcrypt_t *xbcrypt_file;
ds_encrypt_ctxt_t *crypt_ctxt;
size_t bytes_processed;
ds_file_t *dest_file;
} ds_encrypt_file_t;
/* Encryption options */
uint ds_encrypt_encrypt_threads;
ulonglong ds_encrypt_encrypt_chunk_size;
static ds_ctxt_t *encrypt_init(const char *root);
static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path,
MY_STAT *mystat);
static int encrypt_write(ds_file_t *file, const void *buf, size_t len);
static int encrypt_close(ds_file_t *file);
static void encrypt_deinit(ds_ctxt_t *ctxt);
datasink_t datasink_encrypt = {
&encrypt_init,
&encrypt_open,
&encrypt_write,
&encrypt_close,
&encrypt_deinit
};
static crypt_thread_ctxt_t *create_worker_threads(uint n);
static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
static void *encrypt_worker_thread_func(void *arg);
static uint encrypt_iv_len = 0;
static
ssize_t
my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len)
{
ds_encrypt_file_t *encrypt_file;
encrypt_file = (ds_encrypt_file_t *) userdata;
xb_ad(encrypt_file != NULL);
xb_ad(encrypt_file->dest_file != NULL);
if (!ds_write(encrypt_file->dest_file, buf, len)) {
return len;
}
return -1;
}
static
ds_ctxt_t *
encrypt_init(const char *root)
{
ds_ctxt_t *ctxt;
ds_encrypt_ctxt_t *encrypt_ctxt;
crypt_thread_ctxt_t *threads;
if (xb_crypt_init(&encrypt_iv_len)) {
return NULL;
}
/* Create and initialize the worker threads */
threads = create_worker_threads(ds_encrypt_encrypt_threads);
if (threads == NULL) {
msg("encrypt: failed to create worker threads.\n");
return NULL;
}
ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +
sizeof(ds_encrypt_ctxt_t),
MYF(MY_FAE));
encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1);
encrypt_ctxt->threads = threads;
encrypt_ctxt->nthreads = ds_encrypt_encrypt_threads;
ctxt->ptr = encrypt_ctxt;
ctxt->root = my_strdup(root, MYF(MY_FAE));
return ctxt;
}
static
ds_file_t *
encrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
{
ds_ctxt_t *dest_ctxt;
ds_encrypt_ctxt_t *crypt_ctxt;
ds_encrypt_file_t *crypt_file;
char new_name[FN_REFLEN];
ds_file_t *file;
xb_ad(ctxt->pipe_ctxt != NULL);
dest_ctxt = ctxt->pipe_ctxt;
crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr;
file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
sizeof(ds_encrypt_file_t),
MYF(MY_FAE|MY_ZEROFILL));
crypt_file = (ds_encrypt_file_t *) (file + 1);
/* Append the .xbcrypt extension to the filename */
fn_format(new_name, path, "", ".xbcrypt", MYF(MY_APPEND_EXT));
crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat);
if (crypt_file->dest_file == NULL) {
msg("encrypt: ds_open(\"%s\") failed.\n", new_name);
goto err;
}
crypt_file->crypt_ctxt = crypt_ctxt;
crypt_file->xbcrypt_file = xb_crypt_write_open(crypt_file,
my_xb_crypt_write_callback);
if (crypt_file->xbcrypt_file == NULL) {
msg("encrypt: xb_crypt_write_open() failed.\n");
goto err;
}
file->ptr = crypt_file;
file->path = crypt_file->dest_file->path;
return file;
err:
if (crypt_file->dest_file) {
ds_close(crypt_file->dest_file);
}
my_free(file);
return NULL;
}
static
int
encrypt_write(ds_file_t *file, const void *buf, size_t len)
{
ds_encrypt_file_t *crypt_file;
ds_encrypt_ctxt_t *crypt_ctxt;
crypt_thread_ctxt_t *threads;
crypt_thread_ctxt_t *thd;
uint nthreads;
uint i;
const uchar *ptr;
crypt_file = (ds_encrypt_file_t *) file->ptr;
crypt_ctxt = crypt_file->crypt_ctxt;
threads = crypt_ctxt->threads;
nthreads = crypt_ctxt->nthreads;
ptr = (const uchar *) buf;
while (len > 0) {
uint max_thread;
/* Send data to worker threads for encryption */
for (i = 0; i < nthreads; i++) {
size_t chunk_len;
thd = threads + i;
pthread_mutex_lock(&thd->ctrl_mutex);
chunk_len = (len > XB_CRYPT_CHUNK_SIZE) ?
XB_CRYPT_CHUNK_SIZE : len;
thd->from = ptr;
thd->from_len = chunk_len;
pthread_mutex_lock(&thd->data_mutex);
thd->data_avail = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
len -= chunk_len;
if (len == 0) {
break;
}
ptr += chunk_len;
}
max_thread = (i < nthreads) ? i : nthreads - 1;
/* Reap and stream the encrypted data */
for (i = 0; i <= max_thread; i++) {
thd = threads + i;
pthread_mutex_lock(&thd->data_mutex);
while (thd->data_avail == TRUE) {
pthread_cond_wait(&thd->data_cond,
&thd->data_mutex);
}
xb_a(threads[i].to_len > 0);
if (xb_crypt_write_chunk(crypt_file->xbcrypt_file,
threads[i].to,
threads[i].from_len +
XB_CRYPT_HASH_LEN,
threads[i].to_len,
threads[i].iv,
encrypt_iv_len)) {
msg("encrypt: write to the destination file "
"failed.\n");
return 1;
}
crypt_file->bytes_processed += threads[i].from_len;
pthread_mutex_unlock(&threads[i].data_mutex);
pthread_mutex_unlock(&threads[i].ctrl_mutex);
}
}
return 0;
}
static
int
encrypt_close(ds_file_t *file)
{
ds_encrypt_file_t *crypt_file;
ds_file_t *dest_file;
int rc = 0;
crypt_file = (ds_encrypt_file_t *) file->ptr;
dest_file = crypt_file->dest_file;
rc = xb_crypt_write_close(crypt_file->xbcrypt_file);
if (ds_close(dest_file)) {
rc = 1;
}
my_free(file);
return rc;
}
static
void
encrypt_deinit(ds_ctxt_t *ctxt)
{
ds_encrypt_ctxt_t *crypt_ctxt;
xb_ad(ctxt->pipe_ctxt != NULL);
crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr;
destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads);
my_free(ctxt->root);
my_free(ctxt);
}
static
crypt_thread_ctxt_t *
create_worker_threads(uint n)
{
crypt_thread_ctxt_t *threads;
uint i;
threads = (crypt_thread_ctxt_t *)
my_malloc(sizeof(crypt_thread_ctxt_t) * n, MYF(MY_FAE));
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
thd->num = i + 1;
thd->started = FALSE;
thd->cancelled = FALSE;
thd->data_avail = FALSE;
thd->to = (uchar *) my_malloc(XB_CRYPT_CHUNK_SIZE +
XB_CRYPT_HASH_LEN, MYF(MY_FAE));
thd->iv = (uchar *) my_malloc(encrypt_iv_len, MYF(MY_FAE));
/* Initialize the control mutex and condition var */
if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
pthread_cond_init(&thd->ctrl_cond, NULL)) {
goto err;
}
/* Initialize and data mutex and condition var */
if (pthread_mutex_init(&thd->data_mutex, NULL) ||
pthread_cond_init(&thd->data_cond, NULL)) {
goto err;
}
if (xb_crypt_cipher_open(&thd->cipher_handle)) {
goto err;
}
pthread_mutex_lock(&thd->ctrl_mutex);
if (pthread_create(&thd->id, NULL, encrypt_worker_thread_func,
thd)) {
msg("encrypt: pthread_create() failed: "
"errno = %d\n", errno);
goto err;
}
}
/* Wait for the threads to start */
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
while (thd->started == FALSE)
pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);
pthread_mutex_unlock(&thd->ctrl_mutex);
}
return threads;
err:
return NULL;
}
static
void
destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
{
uint i;
for (i = 0; i < n; i++) {
crypt_thread_ctxt_t *thd = threads + i;
pthread_mutex_lock(&thd->data_mutex);
threads[i].cancelled = TRUE;
pthread_cond_signal(&thd->data_cond);
pthread_mutex_unlock(&thd->data_mutex);
pthread_join(thd->id, NULL);
pthread_cond_destroy(&thd->data_cond);
pthread_mutex_destroy(&thd->data_mutex);
pthread_cond_destroy(&thd->ctrl_cond);
pthread_mutex_destroy(&thd->ctrl_mutex);
xb_crypt_cipher_close(thd->cipher_handle);
my_free(thd->to);
my_free(thd->iv);
}
my_free(threads);
}
static
void *
encrypt_worker_thread_func(void *arg)
{
crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg;
pthread_mutex_lock(&thd->ctrl_mutex);
pthread_mutex_lock(&thd->data_mutex);
thd->started = TRUE;
pthread_cond_signal(&thd->ctrl_cond);
pthread_mutex_unlock(&thd->ctrl_mutex);
while (1) {
thd->data_avail = FALSE;
pthread_cond_signal(&thd->data_cond);
while (!thd->data_avail && !thd->cancelled) {
pthread_cond_wait(&thd->data_cond, &thd->data_mutex);
}
if (thd->cancelled)
break;
thd->to_len = thd->from_len;
if (xb_crypt_encrypt(thd->cipher_handle, thd->from,
thd->from_len, thd->to, &thd->to_len,
thd->iv)) {
thd->to_len = 0;
continue;
}
}
pthread_mutex_unlock(&thd->data_mutex);
return NULL;
}
#endif /* HAVE_GCRYPT*/

33
extra/mariabackup/ds_encrypt.h
View File

@ -1,33 +0,0 @@
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
Encryption interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef DS_ENCRYPT_H
#define DS_ENCRYPT_H
#include "datasink.h"
#ifdef HAVE_GCRYPT
extern datasink_t datasink_encrypt;
#endif
/* Encryption options */
extern uint ds_encrypt_encrypt_threads;
extern ulonglong ds_encrypt_encrypt_chunk_size;
#endif

1
extra/mariabackup/encryption_plugin.cc
View File

@ -126,7 +126,6 @@ void encryption_plugin_prepare_init(int argc, char **argv)
if (!xb_plugin_load) if (!xb_plugin_load)
{ {
/* This prevents crashes e.g in --stats with wrong my.cnf*/
finalize_encryption_plugin(0); finalize_encryption_plugin(0);
return; return;
} }

116
extra/mariabackup/innobackupex.cc
View File

@ -101,8 +101,6 @@ char *opt_ibx_login_path = NULL;
ulong opt_ibx_lock_wait_query_type; ulong opt_ibx_lock_wait_query_type;
ulong opt_ibx_kill_long_query_type; ulong opt_ibx_kill_long_query_type;
ulong opt_ibx_decrypt_algo = 0;
uint opt_ibx_kill_long_queries_timeout = 0; uint opt_ibx_kill_long_queries_timeout = 0;
uint opt_ibx_lock_wait_timeout = 0; uint opt_ibx_lock_wait_timeout = 0;
uint opt_ibx_lock_wait_threshold = 0; uint opt_ibx_lock_wait_threshold = 0;
@ -110,7 +108,6 @@ uint opt_ibx_debug_sleep_before_unlock = 0;
uint opt_ibx_safe_slave_backup_timeout = 0; uint opt_ibx_safe_slave_backup_timeout = 0;
const char *opt_ibx_history = NULL; const char *opt_ibx_history = NULL;
bool opt_ibx_decrypt = false;
char *opt_ibx_include = NULL; char *opt_ibx_include = NULL;
char *opt_ibx_databases = NULL; char *opt_ibx_databases = NULL;
@ -121,15 +118,9 @@ char *ibx_backup_directory = NULL;
/* copy of proxied xtrabackup options */ /* copy of proxied xtrabackup options */
my_bool ibx_xb_close_files; my_bool ibx_xb_close_files;
my_bool ibx_xtrabackup_compact;
const char *ibx_xtrabackup_compress_alg; const char *ibx_xtrabackup_compress_alg;
uint ibx_xtrabackup_compress_threads; uint ibx_xtrabackup_compress_threads;
ulonglong ibx_xtrabackup_compress_chunk_size; ulonglong ibx_xtrabackup_compress_chunk_size;
ulong ibx_xtrabackup_encrypt_algo;
char *ibx_xtrabackup_encrypt_key;
char *ibx_xtrabackup_encrypt_key_file;
uint ibx_xtrabackup_encrypt_threads;
ulonglong ibx_xtrabackup_encrypt_chunk_size;
my_bool ibx_xtrabackup_export; my_bool ibx_xtrabackup_export;
char *ibx_xtrabackup_extra_lsndir; char *ibx_xtrabackup_extra_lsndir;
char *ibx_xtrabackup_incremental_basedir; char *ibx_xtrabackup_incremental_basedir;
@ -138,8 +129,6 @@ my_bool ibx_xtrabackup_incremental_force_scan;
ulint ibx_xtrabackup_log_copy_interval; ulint ibx_xtrabackup_log_copy_interval;
char *ibx_xtrabackup_incremental; char *ibx_xtrabackup_incremental;
int ibx_xtrabackup_parallel; int ibx_xtrabackup_parallel;
my_bool ibx_xtrabackup_rebuild_indexes;
ulint ibx_xtrabackup_rebuild_threads;
char *ibx_xtrabackup_stream_str; char *ibx_xtrabackup_stream_str;
char *ibx_xtrabackup_tables_file; char *ibx_xtrabackup_tables_file;
long ibx_xtrabackup_throttle; long ibx_xtrabackup_throttle;
@ -201,7 +190,6 @@ enum innobackupex_options
OPT_NO_VERSION_CHECK, OPT_NO_VERSION_CHECK,
OPT_NO_BACKUP_LOCKS, OPT_NO_BACKUP_LOCKS,
OPT_DATABASES, OPT_DATABASES,
OPT_DECRYPT,
OPT_DECOMPRESS, OPT_DECOMPRESS,
/* options wich are passed directly to xtrabackup */ /* options wich are passed directly to xtrabackup */
@ -210,11 +198,6 @@ enum innobackupex_options
OPT_COMPRESS, OPT_COMPRESS,
OPT_COMPRESS_THREADS, OPT_COMPRESS_THREADS,
OPT_COMPRESS_CHUNK_SIZE, OPT_COMPRESS_CHUNK_SIZE,
OPT_ENCRYPT,
OPT_ENCRYPT_KEY,
OPT_ENCRYPT_KEY_FILE,
OPT_ENCRYPT_THREADS,
OPT_ENCRYPT_CHUNK_SIZE,
OPT_EXPORT, OPT_EXPORT,
OPT_EXTRA_LSNDIR, OPT_EXTRA_LSNDIR,
OPT_INCREMENTAL_BASEDIR, OPT_INCREMENTAL_BASEDIR,
@ -430,12 +413,6 @@ static struct my_option ibx_long_options[] =
(uchar*) &opt_ibx_incremental_history_uuid, 0, GET_STR, (uchar*) &opt_ibx_incremental_history_uuid, 0, GET_STR,
REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"decrypt", OPT_DECRYPT, "Decrypts all files with the .xbcrypt "
"extension in a backup previously made with --encrypt option.",
&opt_ibx_decrypt_algo, &opt_ibx_decrypt_algo,
&xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE,
"This option specifies which types of queries are allowed to complete " "This option specifies which types of queries are allowed to complete "
"before innobackupex will issue the global lock. Default is all.", "before innobackupex will issue the global lock. Default is all.",
@ -533,12 +510,6 @@ static struct my_option ibx_long_options[] =
(uchar*) &ibx_xb_close_files, (uchar*) &ibx_xb_close_files, 0, (uchar*) &ibx_xb_close_files, (uchar*) &ibx_xb_close_files, 0,
GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"compact", OPT_COMPACT, "Create a compact backup with all secondary "
"index pages omitted. This option is passed directly to xtrabackup. "
"See xtrabackup documentation for details.",
(uchar*) &ibx_xtrabackup_compact, (uchar*) &ibx_xtrabackup_compact,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"compress", OPT_COMPRESS, "This option instructs xtrabackup to " {"compress", OPT_COMPRESS, "This option instructs xtrabackup to "
"compress backup copies of InnoDB data files. It is passed directly " "compress backup copies of InnoDB data files. It is passed directly "
"to the xtrabackup child process. Try 'xtrabackup --help' for more " "to the xtrabackup child process. Try 'xtrabackup --help' for more "
@ -560,46 +531,6 @@ static struct my_option ibx_long_options[] =
(uchar*) &ibx_xtrabackup_compress_chunk_size, (uchar*) &ibx_xtrabackup_compress_chunk_size,
0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
{"encrypt", OPT_ENCRYPT, "This option instructs xtrabackup to encrypt "
"backup copies of InnoDB data files using the algorithm specified in "
"the ENCRYPTION-ALGORITHM. It is passed directly to the xtrabackup "
"child process. Try 'xtrabackup --help' for more details.",
&ibx_xtrabackup_encrypt_algo, &ibx_xtrabackup_encrypt_algo,
&xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"encrypt-key", OPT_ENCRYPT_KEY, "This option instructs xtrabackup to "
"use the given ENCRYPTION-KEY when using the --encrypt or --decrypt "
"options. During backup it is passed directly to the xtrabackup child "
"process. Try 'xtrabackup --help' for more details.",
(uchar*) &ibx_xtrabackup_encrypt_key,
(uchar*) &ibx_xtrabackup_encrypt_key, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key-file", OPT_ENCRYPT_KEY_FILE, "This option instructs "
"xtrabackup to use the encryption key stored in the given "
"ENCRYPTION-KEY-FILE when using the --encrypt or --decrypt options.",
(uchar*) &ibx_xtrabackup_encrypt_key_file,
(uchar*) &ibx_xtrabackup_encrypt_key_file, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-threads", OPT_ENCRYPT_THREADS,
"This option specifies the number of worker threads that will be used "
"for parallel encryption. It is passed directly to the xtrabackup "
"child process. Try 'xtrabackup --help' for more details.",
(uchar*) &ibx_xtrabackup_encrypt_threads,
(uchar*) &ibx_xtrabackup_encrypt_threads,
0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0},
{"encrypt-chunk-size", OPT_ENCRYPT_CHUNK_SIZE,
"This option specifies the size of the internal working buffer for "
"each encryption thread, measured in bytes. It is passed directly to "
"the xtrabackup child process. Try 'xtrabackup --help' for more "
"details.",
(uchar*) &ibx_xtrabackup_encrypt_chunk_size,
(uchar*) &ibx_xtrabackup_encrypt_chunk_size,
0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
{"export", OPT_EXPORT, "This option is passed directly to xtrabackup's " {"export", OPT_EXPORT, "This option is passed directly to xtrabackup's "
"--export option. It enables exporting individual tables for import " "--export option. It enables exporting individual tables for import "
"into another server. See the xtrabackup documentation for details.", "into another server. See the xtrabackup documentation for details.",
@ -735,8 +666,6 @@ You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0
SYNOPOSIS\n\ SYNOPOSIS\n\
\n\ \n\
innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chunk-size=CHUNK-SIZE]\n\ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chunk-size=CHUNK-SIZE]\n\
[--encrypt=ENCRYPTION-ALGORITHM] [--encrypt-threads=NUMBER-OF-THREADS] [--encrypt-chunk-size=CHUNK-SIZE]\n\
[--encrypt-key=LITERAL-ENCRYPTION-KEY] | [--encryption-key-file=MY.KEY]\n\
[--include=REGEXP] [--user=NAME]\n\ [--include=REGEXP] [--user=NAME]\n\
[--password=WORD] [--port=PORT] [--socket=SOCKET]\n\ [--password=WORD] [--port=PORT] [--socket=SOCKET]\n\
[--no-timestamp] [--ibbackup=IBBACKUP-BINARY]\n\ [--no-timestamp] [--ibbackup=IBBACKUP-BINARY]\n\
@ -748,7 +677,7 @@ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chu
[--incremental] [--incremental-basedir]\n\ [--incremental] [--incremental-basedir]\n\
[--incremental-dir] [--incremental-force-scan] [--incremental-lsn]\n\ [--incremental-dir] [--incremental-force-scan] [--incremental-lsn]\n\
[--incremental-history-name=NAME] [--incremental-history-uuid=UUID]\n\ [--incremental-history-name=NAME] [--incremental-history-uuid=UUID]\n\
[--close-files] [--compact] \n\ [--close-files]\n\
BACKUP-ROOT-DIR\n\ BACKUP-ROOT-DIR\n\
\n\ \n\
innobackupex --apply-log [--use-memory=B]\n\ innobackupex --apply-log [--use-memory=B]\n\
@ -760,8 +689,7 @@ innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME]
\n\ \n\
innobackupex --move-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ innobackupex --move-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\
\n\ \n\
innobackupex [--decompress] [--decrypt=ENCRYPTION-ALGORITHM]\n\ innobackupex [--decompress]\n\
[--encrypt-key=LITERAL-ENCRYPTION-KEY] | [--encryption-key-file=MY.KEY]\n\
[--parallel=NUMBER-OF-FORKS] BACKUP-DIR\n\ [--parallel=NUMBER-OF-FORKS] BACKUP-DIR\n\
\n\ \n\
DESCRIPTION\n\ DESCRIPTION\n\
@ -798,15 +726,12 @@ it moves files to their original locations rather than copies them. As this\n\
option removes backup files, it must be used with caution. It may be useful in\n\ option removes backup files, it must be used with caution. It may be useful in\n\
cases when there is not enough free disk space to copy files.\n\ cases when there is not enough free disk space to copy files.\n\
\n\ \n\
The --decompress --decrypt command will decrypt and/or decompress a backup made\n\ The --decompress command will decompress a backup made\n\
with the --compress and/or --encrypt options. When decrypting, the encryption\n\ with the --compress option. The\n\
algorithm and key used when the backup was taken MUST be provided via the\n\ --parallel option will allow multiple files to be decompressed\n\
specified options. --decrypt and --decompress may be used together at the same\n\
time to completely normalize a previously compressed and encrypted backup. The\n\
--parallel option will allow multiple files to be decrypted and/or decompressed\n\
simultaneously. In order to decompress, the qpress utility MUST be installed\n\ simultaneously. In order to decompress, the qpress utility MUST be installed\n\
and accessable within the path. This process will remove the original\n\ and accessable within the path. This process will remove the original\n\
compressed/encrypted files and leave the results in the same location.\n\ compressed files and leave the results in the same location.\n\
\n\ \n\
On success the exit code innobackupex is 0. A non-zero exit code \n\ On success the exit code innobackupex is 0. A non-zero exit code \n\
indicates an error.\n"); indicates an error.\n");
@ -839,14 +764,6 @@ ibx_get_one_option(int optid,
opt_ibx_history = ""; opt_ibx_history = "";
} }
break; break;
case OPT_DECRYPT:
if (argument == NULL) {
ibx_msg("Missing --decrypt argument, must specify a "
"valid encryption algorithm.\n");
return(1);
}
opt_ibx_decrypt = true;
break;
case OPT_STREAM: case OPT_STREAM:
if (!strcasecmp(argument, "xbstream")) if (!strcasecmp(argument, "xbstream"))
xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM;
@ -866,15 +783,6 @@ ibx_get_one_option(int optid,
} }
xtrabackup_compress = TRUE; xtrabackup_compress = TRUE;
break; break;
case OPT_ENCRYPT:
if (argument == NULL)
{
msg("Missing --encrypt argument, must specify a "
"valid encryption algorithm.\n");
return 1;
}
xtrabackup_encrypt = TRUE;
break;
case 'p': case 'p':
if (argument) if (argument)
{ {
@ -928,7 +836,7 @@ ibx_handle_options(int *argc, char ***argv)
ibx_mode = IBX_MODE_COPY_BACK; ibx_mode = IBX_MODE_COPY_BACK;
} else if (opt_ibx_move_back) { } else if (opt_ibx_move_back) {
ibx_mode = IBX_MODE_MOVE_BACK; ibx_mode = IBX_MODE_MOVE_BACK;
} else if (opt_ibx_decrypt || opt_ibx_decompress) { } else if (opt_ibx_decompress) {
ibx_mode = IBX_MODE_DECRYPT_DECOMPRESS; ibx_mode = IBX_MODE_DECRYPT_DECOMPRESS;
} else { } else {
ibx_mode = IBX_MODE_BACKUP; ibx_mode = IBX_MODE_BACKUP;
@ -1006,8 +914,6 @@ ibx_init()
opt_lock_wait_query_type = opt_ibx_lock_wait_query_type; opt_lock_wait_query_type = opt_ibx_lock_wait_query_type;
opt_kill_long_query_type = opt_ibx_kill_long_query_type; opt_kill_long_query_type = opt_ibx_kill_long_query_type;
opt_decrypt_algo = opt_ibx_decrypt_algo;
opt_kill_long_queries_timeout = opt_ibx_kill_long_queries_timeout; opt_kill_long_queries_timeout = opt_ibx_kill_long_queries_timeout;
opt_lock_wait_timeout = opt_ibx_lock_wait_timeout; opt_lock_wait_timeout = opt_ibx_lock_wait_timeout;
opt_lock_wait_threshold = opt_ibx_lock_wait_threshold; opt_lock_wait_threshold = opt_ibx_lock_wait_threshold;
@ -1015,18 +921,12 @@ ibx_init()
opt_safe_slave_backup_timeout = opt_ibx_safe_slave_backup_timeout; opt_safe_slave_backup_timeout = opt_ibx_safe_slave_backup_timeout;
opt_history = opt_ibx_history; opt_history = opt_ibx_history;
opt_decrypt = opt_ibx_decrypt;
/* setup xtrabackup options */ /* setup xtrabackup options */
xb_close_files = ibx_xb_close_files; xb_close_files = ibx_xb_close_files;
xtrabackup_compress_alg = ibx_xtrabackup_compress_alg; xtrabackup_compress_alg = ibx_xtrabackup_compress_alg;
xtrabackup_compress_threads = ibx_xtrabackup_compress_threads; xtrabackup_compress_threads = ibx_xtrabackup_compress_threads;
xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size; xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size;
xtrabackup_encrypt_algo = ibx_xtrabackup_encrypt_algo;
xtrabackup_encrypt_key = ibx_xtrabackup_encrypt_key;
xtrabackup_encrypt_key_file = ibx_xtrabackup_encrypt_key_file;
xtrabackup_encrypt_threads = ibx_xtrabackup_encrypt_threads;
xtrabackup_encrypt_chunk_size = ibx_xtrabackup_encrypt_chunk_size;
xtrabackup_export = ibx_xtrabackup_export; xtrabackup_export = ibx_xtrabackup_export;
xtrabackup_extra_lsndir = ibx_xtrabackup_extra_lsndir; xtrabackup_extra_lsndir = ibx_xtrabackup_extra_lsndir;
xtrabackup_incremental_basedir = ibx_xtrabackup_incremental_basedir; xtrabackup_incremental_basedir = ibx_xtrabackup_incremental_basedir;
@ -1107,7 +1007,7 @@ ibx_init()
case IBX_MODE_DECRYPT_DECOMPRESS: case IBX_MODE_DECRYPT_DECOMPRESS:
xtrabackup_decrypt_decompress = TRUE; xtrabackup_decrypt_decompress = TRUE;
xtrabackup_target_dir = ibx_position_arg; xtrabackup_target_dir = ibx_position_arg;
run = "decrypt and decompress"; run = "decompress";
break; break;
default: default:
ut_error; ut_error;

13
extra/mariabackup/xb0xb.h
View File

@ -44,19 +44,6 @@ dberr_t* err, /*!< out: this is set to DB_ERROR if an error
os_file_dir_t dir, /*!< in: directory stream */ os_file_dir_t dir, /*!< in: directory stream */
os_file_stat_t* info) /*!< in/out: buffer where the os_file_stat_t* info) /*!< in/out: buffer where the
info is returned */; info is returned */;
buf_block_t* btr_node_ptr_get_child(
const rec_t* node_ptr,/*!< in: node pointer */
dict_index_t* index, /*!< in: index */
const ulint* offsets,/*!< in: array returned by rec_get_offsets() */
mtr_t* mtr) /*!< in: mtr */;
buf_block_t*
btr_root_block_get(
/*===============*/
const dict_index_t* index, /*!< in: index tree */
ulint mode, /*!< in: either RW_S_LATCH
or RW_X_LATCH */
mtr_t* mtr) /*!< in: mtr */;
fil_space_t* fil_space_t*
fil_space_get_by_name(const char *); fil_space_get_by_name(const char *);
ibool ibool

696
extra/mariabackup/xbcrypt.c
View File

@ -1,696 +0,0 @@
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
The xbcrypt utility: decrypt files in the XBCRYPT format.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#include <my_getopt.h>
#include "common.h"
#include "xbcrypt.h"
#include "xbcrypt_common.h"
#include "crc_glue.h"
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
GCRY_THREAD_OPTION_PTHREAD_IMPL;
#endif
#define XBCRYPT_VERSION "1.1"
typedef enum {
RUN_MODE_NONE,
RUN_MODE_ENCRYPT,
RUN_MODE_DECRYPT
} run_mode_t;
const char *xbcrypt_encrypt_algo_names[] =
{ "NONE", "AES128", "AES192", "AES256", NullS};
TYPELIB xbcrypt_encrypt_algo_typelib=
{array_elements(xbcrypt_encrypt_algo_names)-1,"",
xbcrypt_encrypt_algo_names, NULL};
static run_mode_t opt_run_mode = RUN_MODE_ENCRYPT;
static char *opt_input_file = NULL;
static char *opt_output_file = NULL;
static ulong opt_encrypt_algo;
static char *opt_encrypt_key_file = NULL;
static void *opt_encrypt_key = NULL;
static ulonglong opt_encrypt_chunk_size = 0;
static my_bool opt_verbose = FALSE;
static uint encrypt_algos[] = { GCRY_CIPHER_NONE,
GCRY_CIPHER_AES128,
GCRY_CIPHER_AES192,
GCRY_CIPHER_AES256 };
static int encrypt_algo = 0;
static int encrypt_mode = GCRY_CIPHER_MODE_CTR;
static uint encrypt_key_len = 0;
static size_t encrypt_iv_len = 0;
static struct my_option my_long_options[] =
{
{"help", '?', "Display this help and exit.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"decrypt", 'd', "Decrypt data input to output.",
0, 0, 0,
GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"input", 'i', "Optional input file. If not specified, input"
" will be read from standard input.",
&opt_input_file, &opt_input_file, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"output", 'o', "Optional output file. If not specified, output"
" will be written to standard output.",
&opt_output_file, &opt_output_file, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-algo", 'a', "Encryption algorithm.",
&opt_encrypt_algo, &opt_encrypt_algo, &xbcrypt_encrypt_algo_typelib,
GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key", 'k', "Encryption key.",
&opt_encrypt_key, &opt_encrypt_key, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key-file", 'f', "File which contains encryption key.",
&opt_encrypt_key_file, &opt_encrypt_key_file, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-chunk-size", 's', "Size of working buffer for encryption in"
" bytes. The default value is 64K.",
&opt_encrypt_chunk_size, &opt_encrypt_chunk_size, 0,
GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
{"verbose", 'v', "Display verbose status output.",
&opt_verbose, &opt_verbose,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};
static
int
get_options(int *argc, char ***argv);
static
my_bool
get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
char *argument __attribute__((unused)));
static
void
print_version(void);
static
void
usage(void);
static
int
mode_decrypt(File filein, File fileout);
static
int
mode_encrypt(File filein, File fileout);
int
main(int argc, char **argv)
{
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
gcry_error_t gcry_error;
#endif
File filein = 0;
File fileout = 0;
MY_INIT(argv[0]);
crc_init();
if (get_options(&argc, &argv)) {
goto err;
}
/* Acording to gcrypt docs (and my testing), setting up the threading
callbacks must be done first, so, lets give it a shot */
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
if (gcry_error) {
msg("%s: unable to set libgcrypt thread cbs - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return 1;
}
#endif
/* Version check should be the very first call because it
makes sure that important subsystems are intialized. */
if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
const char *gcrypt_version;
gcrypt_version = gcry_check_version(NULL);
/* No other library has already initialized libgcrypt. */
if (!gcrypt_version) {
msg("%s: failed to initialize libgcrypt\n",
my_progname);
return 1;
} else if (opt_verbose) {
msg("%s: using gcrypt %s\n", my_progname,
gcrypt_version);
}
}
gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
/* Determine the algorithm */
encrypt_algo = encrypt_algos[opt_encrypt_algo];
/* Set up the iv length */
encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
/* Now set up the key */
if (opt_encrypt_key == NULL && opt_encrypt_key_file == NULL) {
msg("%s: no encryption key or key file specified.\n",
my_progname);
return 1;
} else if (opt_encrypt_key && opt_encrypt_key_file) {
msg("%s: both encryption key and key file specified.\n",
my_progname);
return 1;
} else if (opt_encrypt_key_file) {
if (!xb_crypt_read_key_file(opt_encrypt_key_file,
&opt_encrypt_key,
&encrypt_key_len)) {
msg("%s: unable to read encryption key file \"%s\".\n",
opt_encrypt_key_file, my_progname);
return 1;
}
} else {
encrypt_key_len = strlen(opt_encrypt_key);
}
if (opt_input_file) {
MY_STAT mystat;
if (opt_verbose)
msg("%s: input file \"%s\".\n", my_progname,
opt_input_file);
if (my_stat(opt_input_file, &mystat, MYF(MY_WME)) == NULL) {
goto err;
}
if (!MY_S_ISREG(mystat.st_mode)) {
msg("%s: \"%s\" is not a regular file, exiting.\n",
my_progname, opt_input_file);
goto err;
}
if ((filein = my_open(opt_input_file, O_RDONLY, MYF(MY_WME)))
< 0) {
msg("%s: failed to open \"%s\".\n", my_progname,
opt_input_file);
goto err;
}
} else {
if (opt_verbose)
msg("%s: input from standard input.\n", my_progname);
filein = fileno(stdin);
}
if (opt_output_file) {
if (opt_verbose)
msg("%s: output file \"%s\".\n", my_progname,
opt_output_file);
if ((fileout = my_create(opt_output_file, 0,
O_WRONLY|O_BINARY|O_EXCL|O_NOFOLLOW,
MYF(MY_WME))) < 0) {
msg("%s: failed to create output file \"%s\".\n",
my_progname, opt_output_file);
goto err;
}
} else {
if (opt_verbose)
msg("%s: output to standard output.\n", my_progname);
fileout = fileno(stdout);
}
if (opt_run_mode == RUN_MODE_DECRYPT
&& mode_decrypt(filein, fileout)) {
goto err;
} else if (opt_run_mode == RUN_MODE_ENCRYPT
&& mode_encrypt(filein, fileout)) {
goto err;
}
if (opt_input_file && filein) {
my_close(filein, MYF(MY_WME));
}
if (opt_output_file && fileout) {
my_close(fileout, MYF(MY_WME));
}
my_cleanup_options(my_long_options);
my_end(0);
return EXIT_SUCCESS;
err:
if (opt_input_file && filein) {
my_close(filein, MYF(MY_WME));
}
if (opt_output_file && fileout) {
my_close(fileout, MYF(MY_WME));
}
my_cleanup_options(my_long_options);
my_end(0);
exit(EXIT_FAILURE);
}
static
size_t
my_xb_crypt_read_callback(void *userdata, void *buf, size_t len)
{
File* file = (File *) userdata;
return xb_read_full(*file, buf, len);
}
static
int
mode_decrypt(File filein, File fileout)
{
xb_rcrypt_t *xbcrypt_file = NULL;
void *chunkbuf = NULL;
size_t chunksize;
size_t originalsize;
void *ivbuf = NULL;
size_t ivsize;
void *decryptbuf = NULL;
size_t decryptbufsize = 0;
ulonglong ttlchunksread = 0;
ulonglong ttlbytesread = 0;
xb_rcrypt_result_t result;
gcry_cipher_hd_t cipher_handle;
gcry_error_t gcry_error;
my_bool hash_appended;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error = gcry_cipher_open(&cipher_handle,
encrypt_algo,
encrypt_mode, 0);
if (gcry_error) {
msg("%s:decrypt: unable to open libgcrypt"
" cipher - %s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return 1;
}
gcry_error = gcry_cipher_setkey(cipher_handle,
opt_encrypt_key,
encrypt_key_len);
if (gcry_error) {
msg("%s:decrypt: unable to set libgcrypt cipher"
"key - %s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
goto err;
}
}
/* Initialize the xb_crypt format reader */
xbcrypt_file = xb_crypt_read_open(&filein, my_xb_crypt_read_callback);
if (xbcrypt_file == NULL) {
msg("%s:decrypt: xb_crypt_read_open() failed.\n", my_progname);
goto err;
}
/* Walk the encrypted chunks, decrypting them and writing out */
while ((result = xb_crypt_read_chunk(xbcrypt_file, &chunkbuf,
&originalsize, &chunksize,
&ivbuf, &ivsize, &hash_appended))
== XB_CRYPT_READ_CHUNK) {
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("%s:decrypt: unable to reset libgcrypt"
" cipher - %s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
goto err;
}
if (ivsize) {
gcry_error = gcry_cipher_setctr(cipher_handle,
ivbuf,
ivsize);
}
if (gcry_error) {
msg("%s:decrypt: unable to set cipher iv - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
continue;
}
if (decryptbufsize < originalsize) {
decryptbuf = my_realloc(decryptbuf,
originalsize,
MYF(MY_WME | MY_ALLOW_ZERO_PTR));
decryptbufsize = originalsize;
}
/* Try to decrypt it */
gcry_error = gcry_cipher_decrypt(cipher_handle,
decryptbuf,
originalsize,
chunkbuf,
chunksize);
if (gcry_error) {
msg("%s:decrypt: unable to decrypt chunk - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(cipher_handle);
goto err;
}
} else {
decryptbuf = chunkbuf;
}
if (hash_appended) {
uchar hash[XB_CRYPT_HASH_LEN];
originalsize -= XB_CRYPT_HASH_LEN;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_a(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf,
originalsize);
if (memcmp(hash, (char *) decryptbuf + originalsize,
XB_CRYPT_HASH_LEN) != 0) {
msg("%s:%s invalid plaintext hash. "
"Wrong encrytion key specified?\n",
my_progname, __FUNCTION__);
result = XB_CRYPT_READ_ERROR;
goto err;
}
}
/* Write it out */
if (my_write(fileout, (const uchar *) decryptbuf, originalsize,
MYF(MY_WME | MY_NABP))) {
msg("%s:decrypt: unable to write output chunk.\n",
my_progname);
goto err;
}
ttlchunksread++;
ttlbytesread += chunksize;
if (opt_verbose)
msg("%s:decrypt: %llu chunks read, %llu bytes read\n.",
my_progname, ttlchunksread, ttlbytesread);
}
xb_crypt_read_close(xbcrypt_file);
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(cipher_handle);
if (decryptbuf && decryptbufsize)
my_free(decryptbuf);
if (opt_verbose)
msg("\n%s:decrypt: done\n", my_progname);
return 0;
err:
if (xbcrypt_file)
xb_crypt_read_close(xbcrypt_file);
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(cipher_handle);
if (decryptbuf && decryptbufsize)
my_free(decryptbuf);
return 1;
}
static
ssize_t
my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len)
{
File* file = (File *) userdata;
ssize_t ret = my_write(*file, buf, len, MYF(MY_WME));
posix_fadvise(*file, 0, 0, POSIX_FADV_DONTNEED);
return ret;
}
static
int
mode_encrypt(File filein, File fileout)
{
size_t bytesread;
size_t chunkbuflen;
uchar *chunkbuf = NULL;
void *ivbuf = NULL;
size_t encryptbuflen = 0;
size_t encryptedlen = 0;
void *encryptbuf = NULL;
ulonglong ttlchunkswritten = 0;
ulonglong ttlbyteswritten = 0;
xb_wcrypt_t *xbcrypt_file = NULL;
gcry_cipher_hd_t cipher_handle;
gcry_error_t gcry_error;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error = gcry_cipher_open(&cipher_handle,
encrypt_algo,
encrypt_mode, 0);
if (gcry_error) {
msg("%s:encrypt: unable to open libgcrypt cipher - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return 1;
}
gcry_error = gcry_cipher_setkey(cipher_handle,
opt_encrypt_key,
encrypt_key_len);
if (gcry_error) {
msg("%s:encrypt: unable to set libgcrypt cipher key - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
goto err;
}
}
posix_fadvise(filein, 0, 0, POSIX_FADV_SEQUENTIAL);
xbcrypt_file = xb_crypt_write_open(&fileout,
my_xb_crypt_write_callback);
if (xbcrypt_file == NULL) {
msg("%s:encrypt: xb_crypt_write_open() failed.\n",
my_progname);
goto err;
}
ivbuf = my_malloc(encrypt_iv_len, MYF(MY_FAE));
/* now read in data in chunk size, encrypt and write out */
chunkbuflen = opt_encrypt_chunk_size + XB_CRYPT_HASH_LEN;
chunkbuf = (uchar *) my_malloc(chunkbuflen, MYF(MY_FAE));
while ((bytesread = my_read(filein, chunkbuf, opt_encrypt_chunk_size,
MYF(MY_WME))) > 0) {
size_t origbuflen = bytesread + XB_CRYPT_HASH_LEN;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_a(XB_CRYPT_HASH_LEN == gcry_md_get_algo_dlen(XB_CRYPT_HASH));
gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread,
chunkbuf, bytesread);
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("%s:encrypt: unable to reset cipher - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
goto err;
}
xb_crypt_create_iv(ivbuf, encrypt_iv_len);
gcry_error = gcry_cipher_setctr(cipher_handle,
ivbuf,
encrypt_iv_len);
if (gcry_error) {
msg("%s:encrypt: unable to set cipher iv - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
continue;
}
if (encryptbuflen < origbuflen) {
encryptbuf = my_realloc(encryptbuf, origbuflen,
MYF(MY_WME | MY_ALLOW_ZERO_PTR));
encryptbuflen = origbuflen;
}
gcry_error = gcry_cipher_encrypt(cipher_handle,
encryptbuf,
encryptbuflen,
chunkbuf,
origbuflen);
encryptedlen = origbuflen;
if (gcry_error) {
msg("%s:encrypt: unable to encrypt chunk - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(cipher_handle);
goto err;
}
} else {
encryptedlen = origbuflen;
encryptbuf = chunkbuf;
}
if (xb_crypt_write_chunk(xbcrypt_file, encryptbuf,
bytesread + XB_CRYPT_HASH_LEN,
encryptedlen, ivbuf, encrypt_iv_len)) {
msg("%s:encrypt: abcrypt_write_chunk() failed.\n",
my_progname);
goto err;
}
ttlchunkswritten++;
ttlbyteswritten += encryptedlen;
if (opt_verbose)
msg("%s:encrypt: %llu chunks written, %llu bytes "
"written\n.", my_progname, ttlchunkswritten,
ttlbyteswritten);
}
my_free(ivbuf);
my_free(chunkbuf);
if (encryptbuf && encryptbuflen)
my_free(encryptbuf);
xb_crypt_write_close(xbcrypt_file);
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(cipher_handle);
if (opt_verbose)
msg("\n%s:encrypt: done\n", my_progname);
return 0;
err:
if (chunkbuf)
my_free(chunkbuf);
if (encryptbuf && encryptbuflen)
my_free(encryptbuf);
if (xbcrypt_file)
xb_crypt_write_close(xbcrypt_file);
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(cipher_handle);
return 1;
}
static
int
get_options(int *argc, char ***argv)
{
int ho_error;
if ((ho_error= handle_options(argc, argv, my_long_options,
get_one_option))) {
exit(EXIT_FAILURE);
}
return 0;
}
static
my_bool
get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
char *argument __attribute__((unused)))
{
switch (optid) {
case 'd':
opt_run_mode = RUN_MODE_DECRYPT;
break;
case '?':
usage();
exit(0);
}
return FALSE;
}
static
void
print_version(void)
{
printf("%s Ver %s for %s (%s)\n", my_progname, XBCRYPT_VERSION,
SYSTEM_TYPE, MACHINE_TYPE);
}
static
void
usage(void)
{
print_version();
puts("Copyright (C) 2011 Percona Inc.");
puts("This software comes with ABSOLUTELY NO WARRANTY. "
"This is free software,\nand you are welcome to modify and "
"redistribute it under the GPL license.\n");
puts("Encrypt or decrypt files in the XBCRYPT format.\n");
puts("Usage: ");
printf(" %s [OPTIONS...]"
" # read data from specified input, encrypting or decrypting "
" and writing the result to the specified output.\n",
my_progname);
puts("\nOptions:");
my_print_help(my_long_options);
}

79
extra/mariabackup/xbcrypt.h
View File

@ -1,79 +0,0 @@
/******************************************************
Copyright (c) 2011 Percona LLC and/or its affiliates.
Encryption interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef XBCRYPT_H
#define XBCRYPT_H
#include <my_base.h>
#include "common.h"
#define XB_CRYPT_CHUNK_MAGIC1 "XBCRYP01"
#define XB_CRYPT_CHUNK_MAGIC2 "XBCRYP02"
#define XB_CRYPT_CHUNK_MAGIC3 "XBCRYP03" /* must be same size as ^^ */
#define XB_CRYPT_CHUNK_MAGIC_CURRENT XB_CRYPT_CHUNK_MAGIC3
#define XB_CRYPT_CHUNK_MAGIC_SIZE (sizeof(XB_CRYPT_CHUNK_MAGIC1)-1)
#define XB_CRYPT_HASH GCRY_MD_SHA256
#define XB_CRYPT_HASH_LEN 32
/******************************************************************************
Write interface */
typedef struct xb_wcrypt_struct xb_wcrypt_t;
/* Callback on write for i/o, must return # of bytes written or -1 on error */
typedef ssize_t xb_crypt_write_callback(void *userdata,
const void *buf, size_t len);
xb_wcrypt_t *xb_crypt_write_open(void *userdata,
xb_crypt_write_callback *onwrite);
/* Takes buffer, original length, encrypted length iv and iv length, formats
output buffer and calls write callback.
Returns 0 on success, 1 on error */
int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
size_t elen, const void *iv, size_t ivlen);
/* Returns 0 on success, 1 on error */
int xb_crypt_write_close(xb_wcrypt_t *crypt);
/******************************************************************************
Read interface */
typedef struct xb_rcrypt_struct xb_rcrypt_t;
/* Callback on read for i/o, must return # of bytes read or -1 on error */
typedef size_t xb_crypt_read_callback(void *userdata, void *buf, size_t len);
xb_rcrypt_t *xb_crypt_read_open(void *userdata,
xb_crypt_read_callback *onread);
typedef enum {
XB_CRYPT_READ_CHUNK,
XB_CRYPT_READ_INCOMPLETE,
XB_CRYPT_READ_EOF,
XB_CRYPT_READ_ERROR
} xb_rcrypt_result_t;
xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf,
size_t *olen, size_t *elen, void **iv,
size_t *ivlen, my_bool *hash_appended);
int xb_crypt_read_close(xb_rcrypt_t *crypt);
#endif

328
extra/mariabackup/xbcrypt_common.c
View File

@ -1,328 +0,0 @@
/******************************************************
Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
Encryption configuration file interface for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#include "common.h"
#include "xbcrypt.h"
#include "xbcrypt_common.h"
/* Encryption options */
char *ds_encrypt_key = NULL;
char *ds_encrypt_key_file = NULL;
ulong ds_encrypt_algo;
static uint encrypt_key_len;
static uint encrypt_iv_len;
static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
static uint encrypt_algo;
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
GCRY_THREAD_OPTION_PTHREAD_IMPL;
#endif
my_bool
xb_crypt_read_key_file(const char *filename, void** key, uint *keylength)
{
FILE *fp;
if (!(fp = my_fopen(filename, O_RDONLY, MYF(0)))) {
msg("%s:%s: unable to open config file \"%s\", errno(%d)\n",
my_progname, __FUNCTION__, filename, my_errno);
return FALSE;
}
fseek(fp, 0 , SEEK_END);
*keylength = ftell(fp);
rewind(fp);
*key = my_malloc(*keylength, MYF(MY_FAE));
*keylength = fread(*key, 1, *keylength, fp);
my_fclose(fp, MYF(0));
return TRUE;
}
void
xb_crypt_create_iv(void* ivbuf, size_t ivlen)
{
gcry_create_nonce(ivbuf, ivlen);
}
gcry_error_t
xb_crypt_init(uint *iv_len)
{
gcry_error_t gcry_error;
/* Acording to gcrypt docs (and my testing), setting up the threading
callbacks must be done first, so, lets give it a shot */
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
if (gcry_error) {
msg("encryption: unable to set libgcrypt thread cbs - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
#endif
/* Version check should be the very next call because it
makes sure that important subsystems are intialized. */
if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
const char *gcrypt_version;
gcrypt_version = gcry_check_version(NULL);
/* No other library has already initialized libgcrypt. */
if (!gcrypt_version) {
msg("encryption: failed to initialize libgcrypt\n");
return 1;
} else {
msg("encryption: using gcrypt %s\n", gcrypt_version);
}
}
/* Disable the gcry secure memory, not dealing with this for now */
gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
if (gcry_error) {
msg("encryption: unable to disable libgcrypt secmem - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Finalize gcry initialization. */
gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
if (gcry_error) {
msg("encryption: unable to finish libgcrypt initialization - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Determine the algorithm */
encrypt_algo = encrypt_algos[ds_encrypt_algo];
/* Set up the iv length */
encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
xb_a(encrypt_iv_len > 0);
if (iv_len != NULL) {
*iv_len = encrypt_iv_len;
}
/* Now set up the key */
if (ds_encrypt_key == NULL &&
ds_encrypt_key_file == NULL) {
msg("encryption: no encryption key or key file specified.\n");
return gcry_error;
} else if (ds_encrypt_key && ds_encrypt_key_file) {
msg("encryption: both encryption key and key file specified.\n");
return gcry_error;
} else if (ds_encrypt_key_file) {
if (!xb_crypt_read_key_file(ds_encrypt_key_file,
(void**)&ds_encrypt_key,
&encrypt_key_len)) {
msg("encryption: unable to read encryption key file"
" \"%s\".\n", ds_encrypt_key_file);
return gcry_error;
}
} else if (ds_encrypt_key) {
encrypt_key_len = strlen(ds_encrypt_key);
} else {
msg("encryption: no encryption key or key file specified.\n");
return gcry_error;
}
return 0;
}
gcry_error_t
xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle)
{
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_open(cipher_handle,
encrypt_algo,
encrypt_mode, 0);
if (gcry_error) {
msg("encryption: unable to open libgcrypt"
" cipher - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(*cipher_handle);
return gcry_error;
}
gcry_error = gcry_cipher_setkey(*cipher_handle,
ds_encrypt_key,
encrypt_key_len);
if (gcry_error) {
msg("encryption: unable to set libgcrypt"
" cipher key - %s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(*cipher_handle);
return gcry_error;
}
return gcry_error;
}
return 0;
}
void
xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle)
{
if (encrypt_algo != GCRY_CIPHER_NONE)
gcry_cipher_close(cipher_handle);
}
gcry_error_t
xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len,
const uchar *iv, size_t iv_len, my_bool hash_appended)
{
*to_len = from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error_t gcry_error;
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("%s:encryption: unable to reset libgcrypt"
" cipher - %s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
if (iv_len > 0) {
gcry_error = gcry_cipher_setctr(cipher_handle,
iv, iv_len);
}
if (gcry_error) {
msg("%s:encryption: unable to set cipher iv - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
/* Try to decrypt it */
gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len,
from, from_len);
if (gcry_error) {
msg("%s:encryption: unable to decrypt chunk - "
"%s : %s\n", my_progname,
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
gcry_cipher_close(cipher_handle);
return gcry_error;
}
if (hash_appended) {
uchar hash[XB_CRYPT_HASH_LEN];
*to_len -= XB_CRYPT_HASH_LEN;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to,
*to_len);
if (memcmp(hash, (char *) to + *to_len,
XB_CRYPT_HASH_LEN) != 0) {
msg("%s:%s invalid plaintext hash. "
"Wrong encrytion key specified?\n",
my_progname, __FUNCTION__);
return 1;
}
}
} else {
memcpy(to, from, *to_len);
}
return 0;
}
gcry_error_t
xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, uchar *iv)
{
gcry_error_t gcry_error;
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
XB_CRYPT_HASH_LEN);
memcpy(to, from, from_len);
gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len,
from, from_len);
*to_len = from_len;
if (encrypt_algo != GCRY_CIPHER_NONE) {
gcry_error = gcry_cipher_reset(cipher_handle);
if (gcry_error) {
msg("encrypt: unable to reset cipher - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
xb_crypt_create_iv(iv, encrypt_iv_len);
gcry_error = gcry_cipher_setctr(cipher_handle, iv,
encrypt_iv_len);
if (gcry_error) {
msg("encrypt: unable to set cipher ctr - "
"%s : %s\n",
gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
gcry_error = gcry_cipher_encrypt(cipher_handle, to,
*to_len + XB_CRYPT_HASH_LEN,
to,
from_len + XB_CRYPT_HASH_LEN);
if (gcry_error) {
msg("encrypt: unable to encrypt buffer - "
"%s : %s\n", gcry_strsource(gcry_error),
gcry_strerror(gcry_error));
return gcry_error;
}
} else {
memcpy(to, from, from_len + XB_CRYPT_HASH_LEN);
}
*to_len += XB_CRYPT_HASH_LEN;
return 0;
}
#endif

64
extra/mariabackup/xbcrypt_common.h
View File

@ -1,64 +0,0 @@
/******************************************************
Copyright (c) 2017 Percona LLC and/or its affiliates.
Encryption datasink implementation for XtraBackup.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include <my_base.h>
#if HAVE_GCRYPT
#if GCC_VERSION >= 4002
/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
#include <gcrypt.h>
extern char *ds_encrypt_key;
extern char *ds_encrypt_key_file;
extern int ds_encrypt_threads;
extern ulong ds_encrypt_algo;
/******************************************************************************
Utility interface */
my_bool xb_crypt_read_key_file(const char *filename,
void** key, uint *keylength);
void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
/* Initialize gcrypt and setup encryption key and IV lengths */
gcry_error_t
xb_crypt_init(uint *iv_len);
/* Setup gcrypt cipher */
gcry_error_t
xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle);
/* Close gcrypt cipher */
void
xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle);
/* Decrypt buffer */
gcry_error_t
xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, const uchar *iv,
size_t iv_len, my_bool hash_appended);
/* Encrypt buffer */
gcry_error_t
xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
size_t from_len, uchar *to, size_t *to_len, uchar *iv);
#endif

252
extra/mariabackup/xbcrypt_read.c
View File

@ -1,252 +0,0 @@
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
The xbcrypt format reader implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include "xbcrypt.h"
#include "crc_glue.h"
struct xb_rcrypt_struct {
void *userdata;
xb_crypt_read_callback *read;
void *buffer;
size_t bufsize;
void *ivbuffer;
size_t ivbufsize;
ulonglong offset;
};
xb_rcrypt_t *
xb_crypt_read_open(void *userdata, xb_crypt_read_callback *onread)
{
xb_rcrypt_t *crypt;
xb_ad(onread);
crypt = (xb_rcrypt_t *) my_malloc(sizeof(xb_rcrypt_t), MYF(MY_FAE));
crypt->userdata = userdata;
crypt->read = onread;
crypt->buffer = NULL;
crypt->bufsize = 0;
crypt->offset = 0;
crypt->ivbuffer = NULL;
crypt->ivbufsize = 0;
return crypt;
}
xb_rcrypt_result_t
xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen,
void **iv, size_t *ivlen, my_bool *hash_appended)
{
uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4];
uchar *ptr;
ulonglong tmp;
ulong checksum, checksum_exp, version;
size_t bytesread;
xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK;
if ((bytesread = crypt->read(crypt->userdata, tmpbuf, sizeof(tmpbuf)))
!= sizeof(tmpbuf)) {
if (bytesread == 0) {
result = XB_CRYPT_READ_EOF;
goto err;
} else {
msg("%s:%s: unable to read chunk header data at "
"offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
}
ptr = tmpbuf;
if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 3;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 2;
} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1,
XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
version = 1;
} else {
msg("%s:%s: wrong chunk magic at offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
crypt->offset += XB_CRYPT_CHUNK_MAGIC_SIZE;
tmp = uint8korr(ptr); /* reserved */
ptr += 8;
crypt->offset += 8;
tmp = uint8korr(ptr); /* original size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid original size at offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->offset += 8;
*olen = (size_t)tmp;
tmp = uint8korr(ptr); /* encrypted size */
ptr += 8;
if (tmp > INT_MAX) {
msg("%s:%s: invalid encrypted size at offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->offset += 8;
*elen = (size_t)tmp;
checksum_exp = uint4korr(ptr); /* checksum */
ptr += 4;
crypt->offset += 4;
/* iv size */
if (version == 1) {
*ivlen = 0;
*iv = 0;
} else {
if ((bytesread = crypt->read(crypt->userdata, tmpbuf, 8))
!= 8) {
if (bytesread == 0) {
result = XB_CRYPT_READ_EOF;
goto err;
} else {
msg("%s:%s: unable to read chunk iv size at "
"offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
}
tmp = uint8korr(tmpbuf);
if (tmp > INT_MAX) {
msg("%s:%s: invalid iv size at offset 0x%llx.\n",
my_progname, __FUNCTION__, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->offset += 8;
*ivlen = (size_t)tmp;
}
if (*ivlen > crypt->ivbufsize) {
crypt->ivbuffer = my_realloc(crypt->ivbuffer, *ivlen,
MYF(MY_WME | MY_ALLOW_ZERO_PTR));
if (crypt->ivbuffer == NULL) {
msg("%s:%s: failed to increase iv buffer to "
"%llu bytes.\n", my_progname, __FUNCTION__,
(ulonglong)*ivlen);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->ivbufsize = *ivlen;
}
if (*ivlen > 0) {
if (crypt->read(crypt->userdata, crypt->ivbuffer, *ivlen)
!= *ivlen) {
msg("%s:%s: failed to read %lld bytes for chunk iv "
"at offset 0x%llx.\n", my_progname, __FUNCTION__,
(ulonglong)*ivlen, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
*iv = crypt->ivbuffer;
}
/* for version euqals 2 we need to read in the iv data but do not init
CTR with it */
if (version == 2) {
*ivlen = 0;
*iv = 0;
}
if (*olen > crypt->bufsize) {
crypt->buffer = my_realloc(crypt->buffer, *olen,
MYF(MY_WME | MY_ALLOW_ZERO_PTR));
if (crypt->buffer == NULL) {
msg("%s:%s: failed to increase buffer to "
"%llu bytes.\n", my_progname, __FUNCTION__,
(ulonglong)*olen);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->bufsize = *olen;
}
if (*elen > 0) {
if (crypt->read(crypt->userdata, crypt->buffer, *elen)
!= *elen) {
msg("%s:%s: failed to read %lld bytes for chunk payload "
"at offset 0x%llx.\n", my_progname, __FUNCTION__,
(ulonglong)*elen, crypt->offset);
result = XB_CRYPT_READ_ERROR;
goto err;
}
}
checksum = crc32_iso3309(0, crypt->buffer, *elen);
if (checksum != checksum_exp) {
msg("%s:%s invalid checksum at offset 0x%llx, "
"expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__,
crypt->offset, checksum_exp, checksum);
result = XB_CRYPT_READ_ERROR;
goto err;
}
crypt->offset += *elen;
*buf = crypt->buffer;
*hash_appended = version > 2;
goto exit;
err:
*buf = NULL;
*olen = 0;
*elen = 0;
*ivlen = 0;
*iv = 0;
exit:
return result;
}
int xb_crypt_read_close(xb_rcrypt_t *crypt)
{
if (crypt->buffer)
my_free(crypt->buffer);
if (crypt->ivbuffer)
my_free(crypt->ivbuffer);
my_free(crypt);
return 0;
}

105
extra/mariabackup/xbcrypt_write.c
View File

@ -1,105 +0,0 @@
/******************************************************
Copyright (c) 2013 Percona LLC and/or its affiliates.
The xbcrypt format writer implementation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include "xbcrypt.h"
#include "crc_glue.h"
struct xb_wcrypt_struct {
void *userdata;
xb_crypt_write_callback *write;
};
xb_wcrypt_t *
xb_crypt_write_open(void *userdata, xb_crypt_write_callback *onwrite)
{
xb_wcrypt_t *crypt;
xb_ad(onwrite);
crypt = (xb_wcrypt_t *) my_malloc(sizeof(xb_wcrypt_t), MYF(MY_FAE));
crypt->userdata = userdata;
crypt->write = onwrite;
return crypt;
}
int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
size_t elen, const void *iv, size_t ivlen)
{
uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4 + 8];
uchar *ptr;
ulong checksum;
xb_ad(olen <= INT_MAX);
if (olen > INT_MAX)
return 0;
xb_ad(elen <= INT_MAX);
if (elen > INT_MAX)
return 0;
xb_ad(ivlen <= INT_MAX);
if (ivlen > INT_MAX)
return 0;
ptr = tmpbuf;
memcpy(ptr, XB_CRYPT_CHUNK_MAGIC_CURRENT, XB_CRYPT_CHUNK_MAGIC_SIZE);
ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
int8store(ptr, (ulonglong)0); /* reserved */
ptr += 8;
int8store(ptr, (ulonglong)olen); /* original size */
ptr += 8;
int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */
ptr += 8;
checksum = crc32_iso3309(0, buf, elen);
int4store(ptr, checksum); /* checksum */
ptr += 4;
int8store(ptr, (ulonglong)ivlen); /* iv size */
ptr += 8;
xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
if (crypt->write(crypt->userdata, tmpbuf, ptr-tmpbuf) == -1)
return 1;
if (crypt->write(crypt->userdata, iv, ivlen) == -1)
return 1;
if (crypt->write(crypt->userdata, buf, elen) == -1)
return 1;
return 0;
}
int xb_crypt_write_close(xb_wcrypt_t *crypt)
{
my_free(crypt);
return 0;
}

57
extra/mariabackup/xbstream.c
View File

@ -25,9 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_pthread.h> #include <my_pthread.h>
#include "common.h" #include "common.h"
#include "xbstream.h" #include "xbstream.h"
#include "xbcrypt_common.h"
#include "datasink.h" #include "datasink.h"
#include "ds_decrypt.h"
#include "crc_glue.h" #include "crc_glue.h"
#define XBSTREAM_VERSION "1.0" #define XBSTREAM_VERSION "1.0"
@ -41,33 +39,18 @@ typedef enum {
RUN_MODE_EXTRACT RUN_MODE_EXTRACT
} run_mode_t; } run_mode_t;
const char *xbstream_encrypt_algo_names[] =
{ "NONE", "AES128", "AES192", "AES256", NullS};
TYPELIB xbstream_encrypt_algo_typelib=
{array_elements(xbstream_encrypt_algo_names)-1,"",
xbstream_encrypt_algo_names, NULL};
/* Need the following definitions to avoid linking with ds_*.o and their link /* Need the following definitions to avoid linking with ds_*.o and their link
dependencies */ dependencies */
datasink_t datasink_archive; datasink_t datasink_archive;
datasink_t datasink_xbstream; datasink_t datasink_xbstream;
datasink_t datasink_compress; datasink_t datasink_compress;
datasink_t datasink_tmpfile; datasink_t datasink_tmpfile;
datasink_t datasink_encrypt;
datasink_t datasink_buffer; datasink_t datasink_buffer;
static run_mode_t opt_mode; static run_mode_t opt_mode;
static char * opt_directory = NULL; static char * opt_directory = NULL;
static my_bool opt_verbose = 0; static my_bool opt_verbose = 0;
static int opt_parallel = 1; static int opt_parallel = 1;
static ulong opt_encrypt_algo;
static char *opt_encrypt_key_file = NULL;
static void *opt_encrypt_key = NULL;
static int opt_encrypt_threads = 1;
enum {
OPT_ENCRYPT_THREADS = 256
};
static struct my_option my_long_options[] = static struct my_option my_long_options[] =
{ {
@ -86,20 +69,6 @@ static struct my_option my_long_options[] =
{"parallel", 'p', "Number of worker threads for reading / writing.", {"parallel", 'p', "Number of worker threads for reading / writing.",
&opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG, &opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG,
1, 1, INT_MAX, 0, 0, 0}, 1, 1, INT_MAX, 0, 0, 0},
{"decrypt", 'd', "Decrypt files ending with .xbcrypt.",
&opt_encrypt_algo, &opt_encrypt_algo, &xbstream_encrypt_algo_typelib,
GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key", 'k', "Encryption key.",
&opt_encrypt_key, &opt_encrypt_key, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-key-file", 'f', "File which contains encryption key.",
&opt_encrypt_key_file, &opt_encrypt_key_file, 0,
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"encrypt-threads", OPT_ENCRYPT_THREADS,
"Number of threads for parallel data encryption. "
"The default value is 1.",
&opt_encrypt_threads, &opt_encrypt_threads,
0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
}; };
@ -108,7 +77,6 @@ typedef struct {
HASH *filehash; HASH *filehash;
xb_rstream_t *stream; xb_rstream_t *stream;
ds_ctxt_t *ds_ctxt; ds_ctxt_t *ds_ctxt;
ds_ctxt_t *ds_decrypt_ctxt;
pthread_mutex_t *mutex; pthread_mutex_t *mutex;
} extract_ctxt_t; } extract_ctxt_t;
@ -348,19 +316,6 @@ err:
return 1; return 1;
} }
/************************************************************************
Check if string ends with given suffix.
@return true if string ends with given suffix. */
static
my_bool
ends_with(const char *str, const char *suffix)
{
size_t suffix_len = strlen(suffix);
size_t str_len = strlen(str);
return(str_len >= suffix_len
&& strcmp(str + str_len - suffix_len, suffix) == 0);
}
static static
file_entry_t * file_entry_t *
file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
@ -380,11 +335,8 @@ file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
} }
entry->pathlen = pathlen; entry->pathlen = pathlen;
if (ctxt->ds_decrypt_ctxt && ends_with(path, ".xbcrypt")) { file = ds_open(ctxt->ds_ctxt, path, NULL);
file = ds_open(ctxt->ds_decrypt_ctxt, path, NULL);
} else {
file = ds_open(ctxt->ds_ctxt, path, NULL);
}
if (file == NULL) { if (file == NULL) {
msg("%s: failed to create file.\n", my_progname); msg("%s: failed to create file.\n", my_progname);
goto err; goto err;
@ -534,7 +486,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)),
xb_rstream_t *stream = NULL; xb_rstream_t *stream = NULL;
HASH filehash; HASH filehash;
ds_ctxt_t *ds_ctxt = NULL; ds_ctxt_t *ds_ctxt = NULL;
ds_ctxt_t *ds_decrypt_ctxt = NULL;
extract_ctxt_t ctxt; extract_ctxt_t ctxt;
int i; int i;
pthread_t *tids = NULL; pthread_t *tids = NULL;
@ -574,7 +525,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)),
ctxt.stream = stream; ctxt.stream = stream;
ctxt.filehash = &filehash; ctxt.filehash = &filehash;
ctxt.ds_ctxt = ds_ctxt; ctxt.ds_ctxt = ds_ctxt;
ctxt.ds_decrypt_ctxt = ds_decrypt_ctxt;
ctxt.mutex = &mutex; ctxt.mutex = &mutex;
tids = malloc(sizeof(pthread_t) * n_threads); tids = malloc(sizeof(pthread_t) * n_threads);
@ -604,9 +554,6 @@ exit:
if (ds_ctxt != NULL) { if (ds_ctxt != NULL) {
ds_destroy(ds_ctxt); ds_destroy(ds_ctxt);
} }
if (ds_decrypt_ctxt) {
ds_destroy(ds_decrypt_ctxt);
}
xb_stream_read_done(stream); xb_stream_read_done(stream);
return ret; return ret;

808
extra/mariabackup/xtrabackup.cc
View File

File diff suppressed because it is too large Load Diff

12
extra/mariabackup/xtrabackup.h
View File

@ -81,7 +81,6 @@ extern char *xtrabackup_tables_exclude;
extern char *xtrabackup_databases_exclude; extern char *xtrabackup_databases_exclude;
extern ibool xtrabackup_compress; extern ibool xtrabackup_compress;
extern ibool xtrabackup_encrypt;
extern my_bool xtrabackup_backup; extern my_bool xtrabackup_backup;
extern my_bool xtrabackup_prepare; extern my_bool xtrabackup_prepare;
@ -92,15 +91,10 @@ extern my_bool xtrabackup_decrypt_decompress;
extern char *innobase_data_file_path; extern char *innobase_data_file_path;
extern char *innobase_doublewrite_file; extern char *innobase_doublewrite_file;
extern char *xtrabackup_encrypt_key;
extern char *xtrabackup_encrypt_key_file;
extern longlong innobase_log_file_size; extern longlong innobase_log_file_size;
extern long innobase_log_files_in_group; extern long innobase_log_files_in_group;
extern longlong innobase_page_size; extern longlong innobase_page_size;
extern const char *xtrabackup_encrypt_algo_names[];
extern TYPELIB xtrabackup_encrypt_algo_typelib;
extern int xtrabackup_parallel; extern int xtrabackup_parallel;
extern my_bool xb_close_files; extern my_bool xb_close_files;
@ -113,9 +107,6 @@ extern "C"{
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
extern ulong xtrabackup_encrypt_algo;
extern uint xtrabackup_encrypt_threads;
extern ulonglong xtrabackup_encrypt_chunk_size;
extern my_bool xtrabackup_export; extern my_bool xtrabackup_export;
extern char *xtrabackup_incremental_basedir; extern char *xtrabackup_incremental_basedir;
extern char *xtrabackup_extra_lsndir; extern char *xtrabackup_extra_lsndir;
@ -158,8 +149,6 @@ extern TYPELIB query_type_typelib;
extern ulong opt_lock_wait_query_type; extern ulong opt_lock_wait_query_type;
extern ulong opt_kill_long_query_type; extern ulong opt_kill_long_query_type;
extern ulong opt_decrypt_algo;
extern uint opt_kill_long_queries_timeout; extern uint opt_kill_long_queries_timeout;
extern uint opt_lock_wait_timeout; extern uint opt_lock_wait_timeout;
extern uint opt_lock_wait_threshold; extern uint opt_lock_wait_threshold;
@ -167,7 +156,6 @@ extern uint opt_debug_sleep_before_unlock;
extern uint opt_safe_slave_backup_timeout; extern uint opt_safe_slave_backup_timeout;
extern const char *opt_history; extern const char *opt_history;
extern my_bool opt_decrypt;
enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON, enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON,
BINLOG_INFO_AUTO}; BINLOG_INFO_AUTO};

9
mysql-test/r/func_group.result
View File

@ -2448,6 +2448,15 @@ DROP TABLE t1;
# End of 10.1 tests # End of 10.1 tests
# #
# #
# MDEV-13064: assertion `n < m_size' fails in Item::split_sum_func2()
#
create table t1 (i int) engine=MyISAM;
insert into t1 value (1),(2);
select count(*)+sleep(0) from t1;
count(*)+sleep(0)
2
drop table t1;
#
# Start of 10.3 tests # Start of 10.3 tests
# #
# #

18
mysql-test/r/mdl_sync.result
View File

@ -3049,3 +3049,21 @@ SET DEBUG_SYNC= 'RESET';
disconnect con1; disconnect con1;
disconnect con2; disconnect con2;
disconnect con3; disconnect con3;
#
# MDEV-12620 - set lock_wait_timeout = 1;flush tables with read lock;
# lock not released after timeout
#
CREATE TABLE t1(a INT) ENGINE=InnoDB;
SET debug_sync='open_tables_after_open_and_process_table SIGNAL ready WAIT_FOR go';
SELECT * FROM t1;
connect con1,localhost,root,,;
SET debug_sync='now WAIT_FOR ready';
SET lock_wait_timeout=1;
FLUSH TABLES WITH READ LOCK;
ERROR HY000: Lock wait timeout exceeded; try restarting transaction
SET debug_sync='now SIGNAL go';
connection default;
a
SET debug_sync='RESET';
DROP TABLE t1;
disconnect con1;

8
mysql-test/suite/innodb/t/log_file_size.test
View File

@ -1,5 +1,5 @@
# Test resizing the InnoDB redo log. # Test resizing the InnoDB redo log.
--source include/have_innodb.inc --source include/innodb_page_size_small.inc
# Embedded server tests do not support restarting # Embedded server tests do not support restarting
--source include/not_embedded.inc --source include/not_embedded.inc
# DBUG_EXECUTE_IF is needed # DBUG_EXECUTE_IF is needed
@ -24,6 +24,12 @@ call mtr.add_suppression("InnoDB: Unable to open .*ib_logfile0. to check native
FLUSH TABLES; FLUSH TABLES;
--enable_query_log --enable_query_log
--let $restart_parameters= --innodb-thread-concurrency=1 --innodb-log-file-size=1m --innodb-log-files-in-group=2
--source include/restart_mysqld.inc
--let $restart_parameters= --innodb-thread-concurrency=100 --innodb-log-file-size=10M --innodb-log-files-in-group=2
--source include/restart_mysqld.inc
CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB;
BEGIN; BEGIN;
INSERT INTO t1 VALUES (42); INSERT INTO t1 VALUES (42);

1
mysql-test/suite/mariabackup/full_backup.test
View File

@ -13,7 +13,6 @@ INSERT INTO t VALUES(2);
echo # xtrabackup prepare; echo # xtrabackup prepare;
--disable_result_log --disable_result_log
exec $XTRABACKUP --prepare --target-dir=$targetdir; exec $XTRABACKUP --prepare --target-dir=$targetdir;
exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir;
-- source include/restart_and_restore.inc -- source include/restart_and_restore.inc
--enable_result_log --enable_result_log

5
mysql-test/suite/mariabackup/incremental_encrypted.test
View File

@ -27,11 +27,6 @@ echo # Prepare full backup, apply incremental one;
exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir;
exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir; exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir;
# stats also can support encryption, but needs plugin-load and plugin variables, they are stored in backup-my.cnf
# We need to prepare again to create log files though.
exec $XTRABACKUP --prepare --target-dir=$basedir;
exec $XTRABACKUP --defaults-file=$basedir/backup-my.cnf --stats --datadir=$basedir;
echo # Restore and check results; echo # Restore and check results;
let $targetdir=$basedir; let $targetdir=$basedir;
-- source include/restart_and_restore.inc -- source include/restart_and_restore.inc

1
mysql-test/suite/mariabackup/xb_file_key_management.test
View File

@ -19,7 +19,6 @@ echo # xtrabackup prepare;
--disable_result_log --disable_result_log
exec $XTRABACKUP --prepare --target-dir=$targetdir; exec $XTRABACKUP --prepare --target-dir=$targetdir;
exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir ;
-- source include/restart_and_restore.inc -- source include/restart_and_restore.inc
--enable_result_log --enable_result_log

2
mysql-test/suite/plugins/r/show_all_plugins.result
View File

@ -26,7 +26,7 @@ three_attempts NOT INSTALLED AUTHENTICATION dialog_examples.so GPL
two_questions NOT INSTALLED AUTHENTICATION dialog_examples.so GPL two_questions NOT INSTALLED AUTHENTICATION dialog_examples.so GPL
show status like '%libraries%'; show status like '%libraries%';
Variable_name Value Variable_name Value
Opened_plugin_libraries 8 Opened_plugin_libraries 7
show plugins soname where library = 'ha_example.so'; show plugins soname where library = 'ha_example.so';
Name Status Type Library License Name Status Type Library License
EXAMPLE NOT INSTALLED STORAGE ENGINE ha_example.so GPL EXAMPLE NOT INSTALLED STORAGE ENGINE ha_example.so GPL

4
mysql-test/suite/sys_vars/r/sysvars_innodb.result
View File

@ -1778,8 +1778,8 @@ DEFAULT_VALUE 50331648
VARIABLE_SCOPE GLOBAL VARIABLE_SCOPE GLOBAL
VARIABLE_TYPE BIGINT VARIABLE_TYPE BIGINT
VARIABLE_COMMENT Size of each log file in a log group. VARIABLE_COMMENT Size of each log file in a log group.
NUMERIC_MIN_VALUE 4194304 NUMERIC_MIN_VALUE 1048576
NUMERIC_MAX_VALUE 9223372036854775807 NUMERIC_MAX_VALUE 549755813888
NUMERIC_BLOCK_SIZE 1048576 NUMERIC_BLOCK_SIZE 1048576
ENUM_VALUE_LIST NULL ENUM_VALUE_LIST NULL
READ_ONLY YES READ_ONLY YES

11
mysql-test/t/func_group.test
View File

@ -1692,6 +1692,17 @@ DROP TABLE t1;
--echo # End of 10.1 tests --echo # End of 10.1 tests
--echo # --echo #
--echo #
--echo # MDEV-13064: assertion `n < m_size' fails in Item::split_sum_func2()
--echo #
create table t1 (i int) engine=MyISAM;
insert into t1 value (1),(2);
select count(*)+sleep(0) from t1;
drop table t1;
--echo # --echo #
--echo # Start of 10.3 tests --echo # Start of 10.3 tests
--echo # --echo #

24
mysql-test/t/mdl_sync.test
View File

@ -4075,6 +4075,30 @@ disconnect con2;
disconnect con3; disconnect con3;
--echo #
--echo # MDEV-12620 - set lock_wait_timeout = 1;flush tables with read lock;
--echo # lock not released after timeout
--echo #
CREATE TABLE t1(a INT) ENGINE=InnoDB;
SET debug_sync='open_tables_after_open_and_process_table SIGNAL ready WAIT_FOR go';
send SELECT * FROM t1;
connect (con1,localhost,root,,);
SET debug_sync='now WAIT_FOR ready';
# lock_wait_timeout should be 0 in 10.3, so that we don't have to wait at all
SET lock_wait_timeout=1;
--error ER_LOCK_WAIT_TIMEOUT
FLUSH TABLES WITH READ LOCK;
SET debug_sync='now SIGNAL go';
connection default;
reap;
SET debug_sync='RESET';
DROP TABLE t1;
disconnect con1;
# Check that all connections opened by test cases in this file are really # Check that all connections opened by test cases in this file are really
# gone so execution of other tests won't be affected by their presence. # gone so execution of other tests won't be affected by their presence.
--source include/wait_until_count_sessions.inc --source include/wait_until_count_sessions.inc

6
sql/filesort.cc
View File

@ -182,12 +182,6 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort,
outfile= &sort->io_cache; outfile= &sort->io_cache;
/*
Release InnoDB's adaptive hash index latch (if holding) before
running a sort.
*/
ha_release_temporary_latches(thd);
my_b_clear(&tempfile); my_b_clear(&tempfile);
my_b_clear(&buffpek_pointers); my_b_clear(&buffpek_pointers);
buffpek=0; buffpek=0;

38
sql/handler.cc
View File

@ -2019,44 +2019,6 @@ commit_checkpoint_notify_ha(handlerton *hton, void *cookie)
} }
/**
@details
This function should be called when MySQL sends rows of a SELECT result set
or the EOF mark to the client. It releases a possible adaptive hash index
S-latch held by thd in InnoDB and also releases a possible InnoDB query
FIFO ticket to enter InnoDB. To save CPU time, InnoDB allows a thd to
keep them over several calls of the InnoDB handler interface when a join
is executed. But when we let the control to pass to the client they have
to be released because if the application program uses mysql_use_result(),
it may deadlock on the S-latch if the application on another connection
performs another SQL query. In MySQL-4.1 this is even more important because
there a connection can have several SELECT queries open at the same time.
@param thd the thread handle of the current connection
@return
always 0
*/
int ha_release_temporary_latches(THD *thd)
{
Ha_trx_info *info;
/*
Note that below we assume that only transactional storage engines
may need release_temporary_latches(). If this will ever become false,
we could iterate on thd->open_tables instead (and remove duplicates
as if (!seen[hton->slot]) { seen[hton->slot]=1; ... }).
*/
for (info= thd->transaction.stmt.ha_list; info; info= info->next())
{
handlerton *hton= info->ht();
if (hton && hton->release_temporary_latches)
hton->release_temporary_latches(hton, thd);
}
return 0;
}
/** /**
Check if all storage engines used in transaction agree that after Check if all storage engines used in transaction agree that after
rollback to savepoint it is safe to release MDL locks acquired after rollback to savepoint it is safe to release MDL locks acquired after

4
sql/handler.h
View File

@ -1235,7 +1235,6 @@ struct handlerton
enum_binlog_command binlog_command, enum_binlog_command binlog_command,
const char *query, uint query_length, const char *query, uint query_length,
const char *db, const char *table_name); const char *db, const char *table_name);
int (*release_temporary_latches)(handlerton *hton, THD *thd);
/* /*
Get log status. Get log status.
@ -4352,9 +4351,6 @@ int ha_change_key_cache_param(KEY_CACHE *key_cache);
int ha_repartition_key_cache(KEY_CACHE *key_cache); int ha_repartition_key_cache(KEY_CACHE *key_cache);
int ha_change_key_cache(KEY_CACHE *old_key_cache, KEY_CACHE *new_key_cache); int ha_change_key_cache(KEY_CACHE *old_key_cache, KEY_CACHE *new_key_cache);
/* report to InnoDB that control passes to the client */
int ha_release_temporary_latches(THD *thd);
/* transactions: interface to handlerton functions */ /* transactions: interface to handlerton functions */
int ha_start_consistent_snapshot(THD *thd); int ha_start_consistent_snapshot(THD *thd);
int ha_commit_or_rollback_by_xid(XID *xid, bool commit); int ha_commit_or_rollback_by_xid(XID *xid, bool commit);

5
sql/item.cc
View File

@ -561,8 +561,7 @@ Item::Item(THD *thd):
command => we should check thd->lex->current_select on zero (thd->lex command => we should check thd->lex->current_select on zero (thd->lex
can be uninitialised) can be uninitialised)
*/ */
if (thd->lex->current_select && if (thd->lex->current_select)
thd->stmt_arena->is_stmt_prepare_or_first_sp_execute())
{ {
enum_parsing_place place= enum_parsing_place place=
thd->lex->current_select->parsing_place; thd->lex->current_select->parsing_place;
@ -5764,7 +5763,7 @@ bool Item_field::fix_fields(THD *thd, Item **reference)
SELECT_LEX *select= thd->lex->current_select; SELECT_LEX *select= thd->lex->current_select;
thd->change_item_tree(reference, thd->change_item_tree(reference,
select->parsing_place == IN_GROUP_BY && select->context_analysis_place == IN_GROUP_BY &&
alias_name_used ? *rf->ref : rf); alias_name_used ? *rf->ref : rf);
/* /*

2
sql/item_windowfunc.cc
View File

@ -71,7 +71,7 @@ Item_window_func::fix_fields(THD *thd, Item **ref)
{ {
DBUG_ASSERT(fixed == 0); DBUG_ASSERT(fixed == 0);
enum_parsing_place place= thd->lex->current_select->parsing_place; enum_parsing_place place= thd->lex->current_select->context_analysis_place;
if (!(place == SELECT_LIST || place == IN_ORDER_BY)) if (!(place == SELECT_LIST || place == IN_ORDER_BY))
{ {

8
sql/sql_cache.cc
View File

@ -1,5 +1,5 @@
/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. /* Copyright (c) 2000, 2013, Oracle and/or its affiliates.
Copyright (c) 2010, 2013, Monty Program Ab Copyright (c) 2010, 2017, MariaDB Corporation
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -1465,12 +1465,6 @@ def_week_frmt: %lu, in_trans: %d, autocommit: %d",
(int)flags.in_trans, (int)flags.in_trans,
(int)flags.autocommit)); (int)flags.autocommit));
/*
Make InnoDB to release the adaptive hash index latch before
acquiring the query cache mutex.
*/
ha_release_temporary_latches(thd);
/* /*
A table- or a full flush operation can potentially take a long time to A table- or a full flush operation can potentially take a long time to
finish. We choose not to wait for them and skip caching statements finish. We choose not to wait for them and skip caching statements

14
sql/sql_class.cc
View File

@ -2818,13 +2818,6 @@ int select_send::send_data(List<Item> &items)
if (thd->killed == ABORT_QUERY) if (thd->killed == ABORT_QUERY)
DBUG_RETURN(FALSE); DBUG_RETURN(FALSE);
/*
We may be passing the control from mysqld to the client: release the
InnoDB adaptive hash S-latch to avoid thread deadlocks if it was reserved
by thd
*/
ha_release_temporary_latches(thd);
protocol->prepare_for_resend(); protocol->prepare_for_resend();
if (protocol->send_result_set_row(&items)) if (protocol->send_result_set_row(&items))
{ {
@ -2843,13 +2836,6 @@ int select_send::send_data(List<Item> &items)
bool select_send::send_eof() bool select_send::send_eof()
{ {
/*
We may be passing the control from mysqld to the client: release the
InnoDB adaptive hash S-latch to avoid thread deadlocks if it was reserved
by thd
*/
ha_release_temporary_latches(thd);
/* /*
Don't send EOF if we're in error condition (which implies we've already Don't send EOF if we're in error condition (which implies we've already
sent or are sending an error) sent or are sending an error)

5
sql/sql_insert.cc
View File

@ -1,6 +1,6 @@
/* /*
Copyright (c) 2000, 2016, Oracle and/or its affiliates. Copyright (c) 2000, 2016, Oracle and/or its affiliates.
Copyright (c) 2010, 2016, MariaDB Copyright (c) 2010, 2017, MariaDB
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -3753,9 +3753,6 @@ int select_insert::send_data(List<Item> &values)
} }
} }
// Release latches in case bulk insert takes a long time
ha_release_temporary_latches(thd);
error= write_record(thd, table, &info); error= write_record(thd, table, &info);
table->auto_increment_field_not_null= FALSE; table->auto_increment_field_not_null= FALSE;

1
sql/sql_lex.h
View File

@ -911,6 +911,7 @@ public:
/* reserved for exists 2 in */ /* reserved for exists 2 in */
uint select_n_reserved; uint select_n_reserved;
enum_parsing_place parsing_place; /* where we are parsing expression */ enum_parsing_place parsing_place; /* where we are parsing expression */
enum_parsing_place context_analysis_place; /* where we are in prepare */
bool with_sum_func; /* sum function indicator */ bool with_sum_func; /* sum function indicator */
ulong table_join_options; ulong table_join_options;

3
sql/sql_reload.cc
View File

@ -253,7 +253,8 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options,
NOTE: my_error() has been already called by reopen_tables() within NOTE: my_error() has been already called by reopen_tables() within
close_cached_tables(). close_cached_tables().
*/ */
result= 1; thd->global_read_lock.unlock_global_read_lock(thd);
return 1;
} }
if (thd->global_read_lock.make_global_read_lock_block_commit(thd)) // Killed if (thd->global_read_lock.make_global_read_lock_block_commit(thd)) // Killed

33
sql/sql_select.cc
View File

@ -648,17 +648,15 @@ setup_without_group(THD *thd, Ref_ptr_array ref_pointer_array,
thd->lex->allow_sum_func|= (nesting_map)1 << select->nest_level; thd->lex->allow_sum_func|= (nesting_map)1 << select->nest_level;
save_place= thd->lex->current_select->parsing_place; save_place= thd->lex->current_select->context_analysis_place;
thd->lex->current_select->parsing_place= IN_ORDER_BY; thd->lex->current_select->context_analysis_place= IN_ORDER_BY;
res= res || setup_order(thd, ref_pointer_array, tables, fields, all_fields, res= res || setup_order(thd, ref_pointer_array, tables, fields, all_fields,
order); order);
thd->lex->current_select->parsing_place= save_place; thd->lex->allow_sum_func&= ~((nesting_map)1 << select->nest_level);
thd->lex->allow_sum_func&= ~((nesting_map)1 << select->nest_level); thd->lex->current_select->context_analysis_place= IN_GROUP_BY;
save_place= thd->lex->current_select->parsing_place;
thd->lex->current_select->parsing_place= IN_GROUP_BY;
res= res || setup_group(thd, ref_pointer_array, tables, fields, all_fields, res= res || setup_group(thd, ref_pointer_array, tables, fields, all_fields,
group, hidden_group_fields); group, hidden_group_fields);
thd->lex->current_select->parsing_place= save_place; thd->lex->current_select->context_analysis_place= save_place;
thd->lex->allow_sum_func|= (nesting_map)1 << select->nest_level; thd->lex->allow_sum_func|= (nesting_map)1 << select->nest_level;
res= res || setup_windows(thd, ref_pointer_array, tables, fields, all_fields, res= res || setup_windows(thd, ref_pointer_array, tables, fields, all_fields,
win_specs, win_funcs); win_specs, win_funcs);
@ -712,6 +710,7 @@ JOIN::prepare(TABLE_LIST *tables_init,
if (select_lex->handle_derived(thd->lex, DT_PREPARE)) if (select_lex->handle_derived(thd->lex, DT_PREPARE))
DBUG_RETURN(1); DBUG_RETURN(1);
thd->lex->current_select->context_analysis_place= NO_MATTER;
thd->lex->current_select->is_item_list_lookup= 1; thd->lex->current_select->is_item_list_lookup= 1;
/* /*
If we have already executed SELECT, then it have not sense to prevent If we have already executed SELECT, then it have not sense to prevent
@ -801,12 +800,13 @@ JOIN::prepare(TABLE_LIST *tables_init,
ref_ptrs= ref_ptr_array_slice(0); ref_ptrs= ref_ptr_array_slice(0);
enum_parsing_place save_place= thd->lex->current_select->parsing_place; enum_parsing_place save_place=
thd->lex->current_select->parsing_place= SELECT_LIST; thd->lex->current_select->context_analysis_place;
thd->lex->current_select->context_analysis_place= SELECT_LIST;
if (setup_fields(thd, ref_ptrs, fields_list, MARK_COLUMNS_READ, if (setup_fields(thd, ref_ptrs, fields_list, MARK_COLUMNS_READ,
&all_fields, 1)) &all_fields, 1))
DBUG_RETURN(-1); DBUG_RETURN(-1);
thd->lex->current_select->parsing_place= save_place; thd->lex->current_select->context_analysis_place= save_place;
if (setup_without_group(thd, ref_ptrs, tables_list, if (setup_without_group(thd, ref_ptrs, tables_list,
select_lex->leaf_tables, fields_list, select_lex->leaf_tables, fields_list,
@ -22213,14 +22213,16 @@ int setup_order(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables,
List<Item> &fields, List<Item> &all_fields, ORDER *order, List<Item> &fields, List<Item> &all_fields, ORDER *order,
bool from_window_spec) bool from_window_spec)
{ {
enum_parsing_place parsing_place= thd->lex->current_select->parsing_place; enum_parsing_place context_analysis_place=
thd->lex->current_select->context_analysis_place;
thd->where="order clause"; thd->where="order clause";
for (; order; order=order->next) for (; order; order=order->next)
{ {
if (find_order_in_list(thd, ref_pointer_array, tables, order, fields, if (find_order_in_list(thd, ref_pointer_array, tables, order, fields,
all_fields, FALSE, from_window_spec)) all_fields, FALSE, from_window_spec))
return 1; return 1;
if ((*order->item)->with_window_func && parsing_place != IN_ORDER_BY) if ((*order->item)->with_window_func &&
context_analysis_place != IN_ORDER_BY)
{ {
my_error(ER_WINDOW_FUNCTION_IN_WINDOW_SPEC, MYF(0)); my_error(ER_WINDOW_FUNCTION_IN_WINDOW_SPEC, MYF(0));
return 1; return 1;
@ -22262,7 +22264,8 @@ setup_group(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables,
List<Item> &fields, List<Item> &all_fields, ORDER *order, List<Item> &fields, List<Item> &all_fields, ORDER *order,
bool *hidden_group_fields, bool from_window_spec) bool *hidden_group_fields, bool from_window_spec)
{ {
enum_parsing_place parsing_place= thd->lex->current_select->parsing_place; enum_parsing_place context_analysis_place=
thd->lex->current_select->context_analysis_place;
*hidden_group_fields=0; *hidden_group_fields=0;
ORDER *ord; ORDER *ord;
@ -22278,14 +22281,14 @@ setup_group(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables,
all_fields, TRUE, from_window_spec)) all_fields, TRUE, from_window_spec))
return 1; return 1;
(*ord->item)->marker= UNDEF_POS; /* Mark found */ (*ord->item)->marker= UNDEF_POS; /* Mark found */
if ((*ord->item)->with_sum_func && parsing_place == IN_GROUP_BY) if ((*ord->item)->with_sum_func && context_analysis_place == IN_GROUP_BY)
{ {
my_error(ER_WRONG_GROUP_FIELD, MYF(0), (*ord->item)->full_name()); my_error(ER_WRONG_GROUP_FIELD, MYF(0), (*ord->item)->full_name());
return 1; return 1;
} }
if ((*ord->item)->with_window_func) if ((*ord->item)->with_window_func)
{ {
if (parsing_place == IN_GROUP_BY) if (context_analysis_place == IN_GROUP_BY)
my_error(ER_WRONG_PLACEMENT_OF_WINDOW_FUNCTION, MYF(0)); my_error(ER_WRONG_PLACEMENT_OF_WINDOW_FUNCTION, MYF(0));
else else
my_error(ER_WINDOW_FUNCTION_IN_WINDOW_SPEC, MYF(0)); my_error(ER_WINDOW_FUNCTION_IN_WINDOW_SPEC, MYF(0));

7
storage/innobase/btr/btr0bulk.cc
View File

@ -1,6 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
@ -991,11 +992,7 @@ BtrBulk::finish(dberr_t err)
ut_ad(err == DB_SUCCESS); ut_ad(err == DB_SUCCESS);
} }
#ifdef UNIV_DEBUG ut_ad(!sync_check_iterate(dict_sync_check()));
dict_sync_check check(true);
ut_ad(!sync_check_iterate(check));
#endif /* UNIV_DEBUG */
ut_ad(err != DB_SUCCESS || btr_validate_index(m_index, NULL, false)); ut_ad(err != DB_SUCCESS || btr_validate_index(m_index, NULL, false));
return(err); return(err);

11
storage/innobase/buf/buf0flu.cc
View File

@ -1870,15 +1870,8 @@ buf_flush_batch(
counts */ counts */
{ {
ut_ad(flush_type == BUF_FLUSH_LRU || flush_type == BUF_FLUSH_LIST); ut_ad(flush_type == BUF_FLUSH_LRU || flush_type == BUF_FLUSH_LIST);
ut_ad(flush_type == BUF_FLUSH_LRU
#ifdef UNIV_DEBUG || !sync_check_iterate(dict_sync_check()));
{
dict_sync_check check(true);
ut_ad(flush_type != BUF_FLUSH_LIST
|| !sync_check_iterate(check));
}
#endif /* UNIV_DEBUG */
buf_pool_mutex_enter(buf_pool); buf_pool_mutex_enter(buf_pool);

9
storage/innobase/dict/dict0dict.cc
View File

@ -6114,14 +6114,7 @@ dict_set_corrupted(
ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(mutex_own(&dict_sys->mutex));
ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables));
ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes));
ut_ad(!sync_check_iterate(dict_sync_check()));
#ifdef UNIV_DEBUG
{
dict_sync_check check(true);
ut_ad(!sync_check_iterate(check));
}
#endif /* UNIV_DEBUG */
/* Mark the table as corrupted only if the clustered index /* Mark the table as corrupted only if the clustered index
is corrupted */ is corrupted */

146
storage/innobase/handler/ha_innodb.cc
View File

@ -1869,6 +1869,8 @@ void
innobase_srv_conc_exit_innodb( innobase_srv_conc_exit_innodb(
row_prebuilt_t* prebuilt) row_prebuilt_t* prebuilt)
{ {
ut_ad(!sync_check_iterate(sync_check()));
#ifdef WITH_WSREP #ifdef WITH_WSREP
if (wsrep_on(prebuilt->trx->mysql_thd) && if (wsrep_on(prebuilt->trx->mysql_thd) &&
wsrep_thd_is_BF(prebuilt->trx->mysql_thd, FALSE)) { wsrep_thd_is_BF(prebuilt->trx->mysql_thd, FALSE)) {
@ -1877,13 +1879,6 @@ innobase_srv_conc_exit_innodb(
#endif /* WITH_WSREP */ #endif /* WITH_WSREP */
trx_t* trx = prebuilt->trx; trx_t* trx = prebuilt->trx;
#ifdef BTR_CUR_HASH_ADAPT
# ifdef UNIV_DEBUG
btrsea_sync_check check(trx->has_search_latch);
ut_ad(!sync_check_iterate(check));
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
/* This is to avoid making an unnecessary function call. */ /* This is to avoid making an unnecessary function call. */
if (trx->declared_to_be_inside_innodb if (trx->declared_to_be_inside_innodb
@ -1901,13 +1896,7 @@ innobase_srv_conc_force_exit_innodb(
/*================================*/ /*================================*/
trx_t* trx) /*!< in: transaction handle */ trx_t* trx) /*!< in: transaction handle */
{ {
#ifdef BTR_CUR_HASH_ADAPT ut_ad(!sync_check_iterate(sync_check()));
# ifdef UNIV_DEBUG
btrsea_sync_check check(trx->has_search_latch);
ut_ad(!sync_check_iterate(check));
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
/* This is to avoid making an unnecessary function call. */ /* This is to avoid making an unnecessary function call. */
if (trx->declared_to_be_inside_innodb) { if (trx->declared_to_be_inside_innodb) {
@ -1994,13 +1983,7 @@ const char*
thd_innodb_tmpdir( thd_innodb_tmpdir(
THD* thd) THD* thd)
{ {
#ifdef BTR_CUR_HASH_ADAPT ut_ad(!sync_check_iterate(sync_check()));
# ifdef UNIV_DEBUG
trx_t* trx = thd_to_trx(thd);
btrsea_sync_check check(trx->has_search_latch);
ut_ad(!sync_check_iterate(check));
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
const char* tmp_dir = THDVAR(thd, tmpdir); const char* tmp_dir = THDVAR(thd, tmpdir);
@ -2035,28 +2018,6 @@ thd_to_trx_id(
} }
#endif /* WITH_WSREP */ #endif /* WITH_WSREP */
/********************************************************************//**
Call this function when mysqld passes control to the client. That is to
avoid deadlocks on the adaptive hash S-latch possibly held by thd. For more
documentation, see handler.cc.
@return 0 */
inline
int
innobase_release_temporary_latches(
/*===============================*/
handlerton* hton, /*!< in: handlerton */
THD* thd) /*!< in: MySQL thread */
{
DBUG_ASSERT(hton == innodb_hton_ptr);
if (!srv_was_started) {
} else if (trx_t* trx __attribute__((unused))= thd_to_trx(thd)) {
trx_assert_no_search_latch(trx);
}
return(0);
}
/********************************************************************//** /********************************************************************//**
Increments innobase_active_counter and every INNOBASE_WAKE_INTERVALth Increments innobase_active_counter and every INNOBASE_WAKE_INTERVALth
time calls srv_active_wake_master_thread. This function should be used time calls srv_active_wake_master_thread. This function should be used
@ -3384,7 +3345,6 @@ innobase_query_caching_of_table_permitted(
return(false); return(false);
} }
trx_assert_no_search_latch(trx);
innobase_srv_conc_force_exit_innodb(trx); innobase_srv_conc_force_exit_innodb(trx);
if (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN) if (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)
@ -3692,8 +3652,6 @@ ha_innobase::init_table_handle_for_HANDLER(void)
/* Initialize the m_prebuilt struct much like it would be inited in /* Initialize the m_prebuilt struct much like it would be inited in
external_lock */ external_lock */
trx_assert_no_search_latch(m_prebuilt->trx);
innobase_srv_conc_force_exit_innodb(m_prebuilt->trx); innobase_srv_conc_force_exit_innodb(m_prebuilt->trx);
/* If the transaction is not started yet, start it */ /* If the transaction is not started yet, start it */
@ -3913,9 +3871,6 @@ innobase_init(
innobase_hton->flags = innobase_hton->flags =
HTON_SUPPORTS_EXTENDED_KEYS | HTON_SUPPORTS_FOREIGN_KEYS; HTON_SUPPORTS_EXTENDED_KEYS | HTON_SUPPORTS_FOREIGN_KEYS;
innobase_hton->release_temporary_latches =
innobase_release_temporary_latches;
#ifdef MYSQL_REPLACE_TRX_IN_THD #ifdef MYSQL_REPLACE_TRX_IN_THD
innobase_hton->replace_native_transaction_in_thd = innobase_hton->replace_native_transaction_in_thd =
innodb_replace_trx_in_thd; innodb_replace_trx_in_thd;
@ -4808,12 +4763,6 @@ innobase_commit_ordered(
trx = check_trx_exists(thd); trx = check_trx_exists(thd);
TrxInInnoDB trx_in_innodb(trx); TrxInInnoDB trx_in_innodb(trx);
/* Since we will reserve the kernel mutex, we must not be holding the
search system latch, or we will disobey the latching order. But we
already released it in innobase_xa_prepare() (if not before), so just
have an assert here.*/
trx_assert_no_search_latch(trx);
if (!trx_is_registered_for_2pc(trx) && trx_is_started(trx)) { if (!trx_is_registered_for_2pc(trx) && trx_is_started(trx)) {
/* We cannot throw error here; instead we will catch this error /* We cannot throw error here; instead we will catch this error
again in innobase_commit() and report it from there. */ again in innobase_commit() and report it from there. */
@ -5025,12 +4974,6 @@ innobase_rollback_trx(
DBUG_ENTER("innobase_rollback_trx"); DBUG_ENTER("innobase_rollback_trx");
DBUG_PRINT("trans", ("aborting transaction")); DBUG_PRINT("trans", ("aborting transaction"));
/* Release a possible FIFO ticket and search latch. Since we will
reserve the trx_sys->mutex, we have to release the search system
latch first to obey the latching order. */
trx_assert_no_search_latch(trx);
innobase_srv_conc_force_exit_innodb(trx); innobase_srv_conc_force_exit_innodb(trx);
/* If we had reserved the auto-inc lock for some table (if /* If we had reserved the auto-inc lock for some table (if
@ -6540,37 +6483,19 @@ initialize_auto_increment(dict_table_t* table, const Field* field)
dict_table_autoinc_unlock(table); dict_table_autoinc_unlock(table);
} }
/*****************************************************************//** /** Open an InnoDB table
Creates and opens a handle to a table which already exists in an InnoDB @param[in] name table name
database. @return error code
@return 1 if error, 0 if success */ @retval 0 on success */
int int
ha_innobase::open( ha_innobase::open(const char* name, int, uint)
/*==============*/
const char* name, /*!< in: table name */
int mode, /*!< in: not used */
uint test_if_locked) /*!< in: not used */
{ {
dict_table_t* ib_table; dict_table_t* ib_table;
char norm_name[FN_REFLEN]; char norm_name[FN_REFLEN];
THD* thd;
dict_err_ignore_t ignore_err = DICT_ERR_IGNORE_NONE; dict_err_ignore_t ignore_err = DICT_ERR_IGNORE_NONE;
DBUG_ENTER("ha_innobase::open"); DBUG_ENTER("ha_innobase::open");
UT_NOT_USED(mode);
UT_NOT_USED(test_if_locked);
thd = ha_thd();
/* Under some cases MySQL seems to call this function while
holding search latch(es). This breaks the latching order as
we acquire dict_sys->mutex below and leads to a deadlock. */
if (thd != NULL) {
innobase_release_temporary_latches(ht, thd);
}
normalize_table_name(norm_name, name); normalize_table_name(norm_name, name);
m_user_thd = NULL; m_user_thd = NULL;
@ -6585,6 +6510,7 @@ ha_innobase::open(
m_upd_buf_size = 0; m_upd_buf_size = 0;
char* is_part = is_partition(norm_name); char* is_part = is_partition(norm_name);
THD* thd = ha_thd();
/* Check whether FOREIGN_KEY_CHECKS is set to 0. If so, the table /* Check whether FOREIGN_KEY_CHECKS is set to 0. If so, the table
can be opened even if some FK indexes are missing. If not, the table can be opened even if some FK indexes are missing. If not, the table
@ -7055,12 +6981,6 @@ ha_innobase::close()
{ {
DBUG_ENTER("ha_innobase::close"); DBUG_ENTER("ha_innobase::close");
THD* thd = ha_thd();
if (thd != NULL) {
innobase_release_temporary_latches(ht, thd);
}
row_prebuilt_free(m_prebuilt, FALSE); row_prebuilt_free(m_prebuilt, FALSE);
if (m_upd_buf != NULL) { if (m_upd_buf != NULL) {
@ -13156,10 +13076,6 @@ create_table_info_t::initialize()
parent_trx = check_trx_exists(m_thd); parent_trx = check_trx_exists(m_thd);
/* In case MySQL calls this in the middle of a SELECT query, release
possible adaptive hash latch to avoid deadlocks of threads */
trx_assert_no_search_latch(parent_trx);
DBUG_RETURN(0); DBUG_RETURN(0);
} }
@ -14012,12 +13928,6 @@ innobase_drop_database(
THD* thd = current_thd; THD* thd = current_thd;
/* In case MySQL calls this in the middle of a SELECT
query, release possible adaptive hash latch to avoid
deadlocks of threads */
trx_assert_no_search_latch(check_trx_exists(thd));
ulint len = 0; ulint len = 0;
char* ptr = strend(path) - 2; char* ptr = strend(path) - 2;
@ -14751,12 +14661,7 @@ ha_innobase::info_low(
update_thd(ha_thd()); update_thd(ha_thd());
/* In case MySQL calls this in the middle of a SELECT query, release m_prebuilt->trx->op_info = "returning various info to MariaDB";
possible adaptive hash latch to avoid deadlocks of threads */
m_prebuilt->trx->op_info = (char*)"returning various info to MariaDB";
trx_assert_no_search_latch(m_prebuilt->trx);
ib_table = m_prebuilt->table; ib_table = m_prebuilt->table;
DBUG_ASSERT(ib_table->n_ref_count > 0); DBUG_ASSERT(ib_table->n_ref_count > 0);
@ -15603,12 +15508,7 @@ ha_innobase::update_table_comment(
update_thd(ha_thd()); update_thd(ha_thd());
m_prebuilt->trx->op_info = (char*)"returning table comment"; m_prebuilt->trx->op_info = "returning table comment";
/* In case MySQL calls this in the middle of a SELECT query, release
possible adaptive hash latch to avoid deadlocks of threads */
trx_assert_no_search_latch(m_prebuilt->trx);
#define SSTR( x ) reinterpret_cast< std::ostringstream & >( \ #define SSTR( x ) reinterpret_cast< std::ostringstream & >( \
( std::ostringstream() << std::dec << x ) ).str() ( std::ostringstream() << std::dec << x ) ).str()
@ -15664,22 +15564,14 @@ ha_innobase::get_foreign_key_create_info(void)
update_thd(ha_thd()); update_thd(ha_thd());
m_prebuilt->trx->op_info = (char*)"getting info on foreign keys"; m_prebuilt->trx->op_info = "getting info on foreign keys";
/* In case MySQL calls this in the middle of a SELECT query,
release possible adaptive hash latch to avoid
deadlocks of threads */
trx_assert_no_search_latch(m_prebuilt->trx);
/* Output the data to a temporary string */ /* Output the data to a temporary string */
std::string str = dict_print_info_on_foreign_keys( std::string str = dict_print_info_on_foreign_keys(
TRUE, m_prebuilt->trx, TRUE, m_prebuilt->trx,
m_prebuilt->table); m_prebuilt->table);
m_prebuilt->trx->op_info = (char*)""; m_prebuilt->trx->op_info = "";
/* Allocate buffer for the string */ /* Allocate buffer for the string */
char* fk_str = (char*) my_malloc(str.length() + 1, MYF(0)); char* fk_str = (char*) my_malloc(str.length() + 1, MYF(0));
@ -16682,8 +16574,6 @@ innodb_show_status(
trx_t* trx = check_trx_exists(thd); trx_t* trx = check_trx_exists(thd);
trx_assert_no_search_latch(trx);
innobase_srv_conc_force_exit_innodb(trx); innobase_srv_conc_force_exit_innodb(trx);
TrxInInnoDB trx_in_innodb(trx); TrxInInnoDB trx_in_innodb(trx);
@ -17999,12 +17889,6 @@ innobase_xa_prepare(
thd_get_xid(thd, (MYSQL_XID*) trx->xid); thd_get_xid(thd, (MYSQL_XID*) trx->xid);
/* Release a possible FIFO ticket and search latch. Since we will
reserve the trx_sys->mutex, we have to release the search system
latch first to obey the latching order. */
trx_assert_no_search_latch(trx);
innobase_srv_conc_force_exit_innodb(trx); innobase_srv_conc_force_exit_innodb(trx);
TrxInInnoDB trx_in_innodb(trx); TrxInInnoDB trx_in_innodb(trx);
@ -21333,7 +21217,7 @@ static MYSQL_SYSVAR_LONG(log_buffer_size, innobase_log_buffer_size,
static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size, static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"Size of each log file in a log group.", "Size of each log file in a log group.",
NULL, NULL, 48*1024*1024L, 4*1024*1024L, LLONG_MAX, 1024*1024L); NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, 1 << 20);
static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files, static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,

12
storage/innobase/handler/ha_innodb.h
View File

@ -906,18 +906,6 @@ innodb_base_col_setup_for_stored(
/** whether this is a computed virtual column */ /** whether this is a computed virtual column */
#define innobase_is_v_fld(field) ((field)->vcol_info && !(field)->stored_in_db()) #define innobase_is_v_fld(field) ((field)->vcol_info && !(field)->stored_in_db())
/** Release temporary latches.
Call this function when mysqld passes control to the client. That is to
avoid deadlocks on the adaptive hash S-latch possibly held by thd. For more
documentation, see handler.cc.
@param[in] hton Handlerton.
@param[in] thd MySQL thread.
@return 0 */
int
innobase_release_temporary_latches(
handlerton* hton,
THD* thd);
/** Always normalize table name to lower case on Windows */ /** Always normalize table name to lower case on Windows */
#ifdef _WIN32 #ifdef _WIN32
#define normalize_table_name(norm_name, name) \ #define normalize_table_name(norm_name, name) \

31
storage/innobase/handler/ha_innopart.cc
View File

@ -932,7 +932,6 @@ ha_innopart::open(
{ {
dict_table_t* ib_table; dict_table_t* ib_table;
char norm_name[FN_REFLEN]; char norm_name[FN_REFLEN];
THD* thd;
DBUG_ENTER("ha_innopart::open"); DBUG_ENTER("ha_innopart::open");
@ -942,16 +941,11 @@ ha_innopart::open(
ut_ad(table->part_info != NULL); ut_ad(table->part_info != NULL);
m_part_info = table->part_info; m_part_info = table->part_info;
} }
thd = ha_thd();
/* Under some cases MySQL seems to call this function while /* Under some cases MySQL seems to call this function while
holding search latch(es). This breaks the latching order as holding search latch(es). This breaks the latching order as
we acquire dict_sys->mutex below and leads to a deadlock. */ we acquire dict_sys->mutex below and leads to a deadlock. */
if (thd != NULL) {
innobase_release_temporary_latches(ht, thd);
}
normalize_table_name(norm_name, name); normalize_table_name(norm_name, name);
m_user_thd = NULL; m_user_thd = NULL;
@ -1017,6 +1011,7 @@ share_error:
MONITOR_INC(MONITOR_TABLE_OPEN); MONITOR_INC(MONITOR_TABLE_OPEN);
bool no_tablespace; bool no_tablespace;
THD* thd = ha_thd();
/* TODO: Should we do this check for every partition during ::open()? */ /* TODO: Should we do this check for every partition during ::open()? */
/* TODO: refactor this in ha_innobase so it can increase code reuse. */ /* TODO: refactor this in ha_innobase so it can increase code reuse. */
@ -1372,15 +1367,8 @@ void ha_innopart::clear_ins_upd_nodes()
int int
ha_innopart::close() ha_innopart::close()
{ {
THD* thd;
DBUG_ENTER("ha_innopart::close"); DBUG_ENTER("ha_innopart::close");
thd = ha_thd();
if (thd != NULL) {
innobase_release_temporary_latches(ht, thd);
}
ut_ad(m_pcur_parts == NULL); ut_ad(m_pcur_parts == NULL);
ut_ad(m_clust_pcur_parts == NULL); ut_ad(m_clust_pcur_parts == NULL);
close_partitioning(); close_partitioning();
@ -3032,11 +3020,6 @@ ha_innopart::records_in_range(
m_prebuilt->trx->op_info = (char*)"estimating records in index range"; m_prebuilt->trx->op_info = (char*)"estimating records in index range";
/* In case MySQL calls this in the middle of a SELECT query, release
possible adaptive hash latch to avoid deadlocks of threads. */
trx_assert_no_search_latch(m_prebuilt->trx);
active_index = keynr; active_index = keynr;
key = table->key_info + active_index; key = table->key_info + active_index;
@ -3171,11 +3154,6 @@ ha_innopart::estimate_rows_upper_bound()
m_prebuilt->trx->op_info = "calculating upper bound for table rows"; m_prebuilt->trx->op_info = "calculating upper bound for table rows";
/* In case MySQL calls this in the middle of a SELECT query, release
possible adaptive hash latch to avoid deadlocks of threads. */
trx_assert_no_search_latch(m_prebuilt->trx);
for (uint i = m_part_info->get_first_used_partition(); for (uint i = m_part_info->get_first_used_partition();
i < m_tot_parts; i < m_tot_parts;
i = m_part_info->get_next_used_partition(i)) { i = m_part_info->get_next_used_partition(i)) {
@ -3287,12 +3265,7 @@ ha_innopart::info_low(
update_thd(ha_thd()); update_thd(ha_thd());
/* In case MySQL calls this in the middle of a SELECT query, release m_prebuilt->trx->op_info = "returning various info to MySQL";
possible adaptive hash latch to avoid deadlocks of threads. */
m_prebuilt->trx->op_info = (char*)"returning various info to MySQL";
trx_assert_no_search_latch(m_prebuilt->trx);
ut_ad(m_part_share->get_table_part(0)->n_ref_count > 0); ut_ad(m_part_share->get_table_part(0)->n_ref_count > 0);

1
storage/innobase/handler/handler0alter.cc
View File

@ -591,7 +591,6 @@ ha_innobase::check_if_supported_inplace_alter(
} }
update_thd(); update_thd();
trx_assert_no_search_latch(m_prebuilt->trx);
/* Change on engine specific table options require rebuild of the /* Change on engine specific table options require rebuild of the
table */ table */

3
storage/innobase/handler/i_s.cc
View File

@ -699,8 +699,7 @@ fill_innodb_trx_from_cache(
#ifdef BTR_CUR_HASH_ADAPT #ifdef BTR_CUR_HASH_ADAPT
/* trx_adaptive_hash_latched */ /* trx_adaptive_hash_latched */
OK(fields[IDX_TRX_ADAPTIVE_HASH_LATCHED]->store( OK(fields[IDX_TRX_ADAPTIVE_HASH_LATCHED]->store(0, true));
row->trx_has_search_latch, true));
#endif /* BTR_CUR_HASH_ADAPT */ #endif /* BTR_CUR_HASH_ADAPT */
/* trx_is_read_only*/ /* trx_is_read_only*/

12
storage/innobase/include/log0log.ic
View File

@ -1,6 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
@ -479,11 +480,11 @@ void
log_free_check(void) log_free_check(void)
/*================*/ /*================*/
{ {
#ifdef UNIV_DEBUG
/* During row_log_table_apply(), this function will be called while we /* During row_log_table_apply(), this function will be called while we
are holding some latches. This is OK, as long as we are not holding are holding some latches. This is OK, as long as we are not holding
any latches on buffer blocks. */ any latches on buffer blocks. */
#ifdef UNIV_DEBUG
static const latch_level_t latches[] = { static const latch_level_t latches[] = {
SYNC_DICT, /* dict_sys->mutex during SYNC_DICT, /* dict_sys->mutex during
commit_try_rebuild() */ commit_try_rebuild() */
@ -491,13 +492,12 @@ log_free_check(void)
commit_try_rebuild() */ commit_try_rebuild() */
SYNC_INDEX_TREE /* index->lock */ SYNC_INDEX_TREE /* index->lock */
}; };
sync_allowed_latches check(
latches, latches + sizeof(latches)/sizeof(*latches));
ut_ad(!sync_check_iterate(check));
#endif /* UNIV_DEBUG */ #endif /* UNIV_DEBUG */
ut_ad(!sync_check_iterate(
sync_allowed_latches(latches,
latches + UT_ARR_SIZE(latches))));
if (log_sys->check_flush_or_checkpoint) { if (log_sys->check_flush_or_checkpoint) {
log_check_margins(); log_check_margins();

8
storage/innobase/include/sync0debug.h
View File

@ -1,7 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 2013, 2015, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. Copyright (c) 2017, MariaDB Corporation.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described Google, Inc. Those modifications are gratefully acknowledged and are described
@ -84,10 +84,10 @@ sync_check_find(latch_level_t level);
/** Checks that the level array for the current thread is empty. /** Checks that the level array for the current thread is empty.
Terminate iteration if the functor returns true. Terminate iteration if the functor returns true.
@param[in,out] functor called for each element. @param[in] functor called for each element.
@return true if the functor returns true */ @return true if the functor returns true for any element */
bool bool
sync_check_iterate(sync_check_functor_t& functor); sync_check_iterate(const sync_check_functor_t& functor);
/** Acquires the debug mutex. We cannot use the mutex defined in sync0sync, /** Acquires the debug mutex. We cannot use the mutex defined in sync0sync,
because the debug mutex is also acquired in sync0arr while holding the OS because the debug mutex is also acquired in sync0arr while holding the OS

163
storage/innobase/include/sync0types.h
View File

@ -1078,108 +1078,43 @@ struct latch_t {
/** Subclass this to iterate over a thread's acquired latch levels. */ /** Subclass this to iterate over a thread's acquired latch levels. */
struct sync_check_functor_t { struct sync_check_functor_t {
virtual ~sync_check_functor_t() { } virtual bool operator()(const latch_level_t) const = 0;
virtual bool operator()(const latch_level_t) = 0;
virtual bool result() const = 0;
}; };
#ifdef BTR_CUR_HASH_ADAPT /** Check that no latch is being held.
/** Functor to check whether the calling thread owns the btr search mutex. */ @tparam some_allowed whether some latches are allowed to be held */
struct btrsea_sync_check : public sync_check_functor_t { template<bool some_allowed = false>
struct sync_checker : public sync_check_functor_t
/** Constructor {
@param[in] has_search_latch true if owns the latch */
explicit btrsea_sync_check(bool has_search_latch)
:
m_result(),
m_has_search_latch(has_search_latch) { }
/** Destructor */
virtual ~btrsea_sync_check() { }
/** Called for every latch owned by the calling thread.
@param[in] level Level of the existing latch
@return true if the predicate check is successful */
virtual bool operator()(const latch_level_t level)
{
/* If calling thread doesn't hold search latch then
check if there are latch level exception provided. */
if (!m_has_search_latch
&& (level != SYNC_SEARCH_SYS
&& level != SYNC_FTS_CACHE)) {
m_result = true;
return(m_result);
}
return(false);
}
/** @return result from the check */
virtual bool result() const
{
return(m_result);
}
private:
/** True if all OK */
bool m_result;
/** If the caller owns the search latch */
const bool m_has_search_latch;
};
#endif /* BTR_CUR_HASH_ADAPT */
/** Functor to check for dictionay latching constraints. */
struct dict_sync_check : public sync_check_functor_t {
/** Constructor
@param[in] dict_mutex_allow true if the dict mutex
is allowed */
explicit dict_sync_check(bool dict_mutex_allowed)
:
m_result(),
m_dict_mutex_allowed(dict_mutex_allowed) { }
/** Destructor */
virtual ~dict_sync_check() { }
/** Check the latching constraints /** Check the latching constraints
@param[in] level The level held by the thread */ @param[in] level The level held by the thread
virtual bool operator()(const latch_level_t level) @return whether a latch violation was detected */
bool operator()(const latch_level_t level) const
{ {
if (!m_dict_mutex_allowed if (some_allowed) {
|| (level != SYNC_DICT switch (level) {
&& level != SYNC_DICT_OPERATION case SYNC_RECV_WRITER:
&& level != SYNC_FTS_CACHE /* This only happens in
/* This only happens in recv_apply_hashed_log_recs. */ recv_apply_hashed_log_recs. */
&& level != SYNC_RECV_WRITER case SYNC_DICT:
&& level != SYNC_NO_ORDER_CHECK)) { case SYNC_DICT_OPERATION:
case SYNC_FTS_CACHE:
m_result = true; case SYNC_NO_ORDER_CHECK:
return(false);
return(true); default:
return(true);
}
} }
return(false); return(true);
} }
/** @return the result of the check */
virtual bool result() const
{
return(m_result);
}
private:
/** True if all OK */
bool m_result;
/** True if it is OK to hold the dict mutex */
const bool m_dict_mutex_allowed;
}; };
/** The strict latch checker (no InnoDB latches may be held) */
typedef struct sync_checker<false> sync_check;
/** The sloppy latch checker (can hold InnoDB dictionary or SQL latches) */
typedef struct sync_checker<true> dict_sync_check;
/** Functor to check for given latching constraints. */ /** Functor to check for given latching constraints. */
struct sync_allowed_latches : public sync_check_functor_t { struct sync_allowed_latches : public sync_check_functor_t {
@ -1189,9 +1124,7 @@ struct sync_allowed_latches : public sync_check_functor_t {
sync_allowed_latches( sync_allowed_latches(
const latch_level_t* from, const latch_level_t* from,
const latch_level_t* to) const latch_level_t* to)
: : begin(from), end(to) { }
m_result(),
m_latches(from, to) { }
/** Checks whether the given latch_t violates the latch constraint. /** Checks whether the given latch_t violates the latch constraint.
This object maintains a list of allowed latch levels, and if the given This object maintains a list of allowed latch levels, and if the given
@ -1199,41 +1132,17 @@ struct sync_allowed_latches : public sync_check_functor_t {
then it is a violation. then it is a violation.
@param[in] latch The latch level to check @param[in] latch The latch level to check
@return true if there is a latch ordering violation */ @return true if there is a latch violation */
virtual bool operator()(const latch_level_t level) bool operator()(const latch_level_t level) const
{ {
for (latches_t::const_iterator it = m_latches.begin(); return(std::find(begin, end, level) == end);
it != m_latches.end();
++it) {
if (level == *it) {
m_result = false;
/* No violation */
return(false);
}
}
return(true);
}
/** @return the result of the check */
virtual bool result() const
{
return(m_result);
} }
private: private:
/** Save the result of validation check here /** First element in an array of allowed latch levels */
True if all OK */ const latch_level_t* const begin;
bool m_result; /** First element after the end of the array of allowed latch levels */
const latch_level_t* const end;
typedef std::vector<latch_level_t, ut_allocator<latch_level_t> >
latches_t;
/** List of latch levels that are allowed to be held */
latches_t m_latches;
}; };
/** Get the latch id from a latch name. /** Get the latch id from a latch name.

5
storage/innobase/include/trx0i_s.h
View File

@ -1,6 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 2007, 2015, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2007, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
@ -162,10 +163,6 @@ struct i_s_trx_row_t {
/*!< check_foreigns in trx_t */ /*!< check_foreigns in trx_t */
const char* trx_foreign_key_error; const char* trx_foreign_key_error;
/*!< detailed_error in trx_t */ /*!< detailed_error in trx_t */
#ifdef BTR_CUR_HASH_ADAPT
ibool trx_has_search_latch;
/*!< has_search_latch in trx_t */
#endif /* BTR_CUR_HASH_ADAPT */
ulint trx_is_read_only; ulint trx_is_read_only;
/*!< trx_t::read_only */ /*!< trx_t::read_only */
ulint trx_is_autocommit_non_locking; ulint trx_is_autocommit_non_locking;

30
storage/innobase/include/trx0trx.h
View File

@ -58,15 +58,6 @@ class FlushObserver;
/** Dummy session used currently in MySQL interface */ /** Dummy session used currently in MySQL interface */
extern sess_t* trx_dummy_sess; extern sess_t* trx_dummy_sess;
#ifdef BTR_CUR_HASH_ADAPT
/** Assert that the transaction is not holding the adaptive hash index latch.
@param[in] trx transaction */
# define trx_assert_no_search_latch(trx) \
ut_ad(!trx->has_search_latch)
#else /* BTR_CUR_HASH_ADAPT */
# define trx_assert_no_search_latch(trx)
#endif
/** Set flush observer for the transaction /** Set flush observer for the transaction
@param[in/out] trx transaction struct @param[in/out] trx transaction struct
@param[in] observer flush observer */ @param[in] observer flush observer */
@ -1072,11 +1063,6 @@ struct trx_t {
flush the log in flush the log in
trx_commit_complete_for_mysql() */ trx_commit_complete_for_mysql() */
ulint duplicates; /*!< TRX_DUP_IGNORE | TRX_DUP_REPLACE */ ulint duplicates; /*!< TRX_DUP_IGNORE | TRX_DUP_REPLACE */
#ifdef BTR_CUR_HASH_ADAPT
bool has_search_latch;
/*!< TRUE if this trx has latched the
search system latch in S-mode */
#endif /* BTR_CUR_HASH_ADAPT */
trx_dict_op_t dict_operation; /**< @see enum trx_dict_op_t */ trx_dict_op_t dict_operation; /**< @see enum trx_dict_op_t */
/* Fields protected by the srv_conc_mutex. */ /* Fields protected by the srv_conc_mutex. */
@ -1508,17 +1494,11 @@ private:
} }
/* Avoid excessive mutex acquire/release */ /* Avoid excessive mutex acquire/release */
if (++trx->in_depth > 1) { if (trx->in_depth++) {
/* The transaction is already inside InnoDB. */ /* The transaction is already inside InnoDB. */
ut_ad(trx->in_depth > 1);
return; return;
} }
/* Only the owning thread should release the latch. */
ut_ad(trx->in_depth == 1);
trx_assert_no_search_latch(trx);
trx_mutex_enter(trx); trx_mutex_enter(trx);
wait(trx); wait(trx);
@ -1543,16 +1523,10 @@ private:
ut_ad(trx->in_depth > 0); ut_ad(trx->in_depth > 0);
if (--trx->in_depth > 0) { if (--trx->in_depth) {
ut_ad(trx->in_depth);
return; return;
} }
/* Only the owning thread should release the latch. */
ut_ad(trx->in_depth == 0);
trx_assert_no_search_latch(trx);
trx_mutex_enter(trx); trx_mutex_enter(trx);
ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK) > 0); ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK) > 0);

38
storage/innobase/log/log0log.cc
View File

@ -677,14 +677,13 @@ log_set_capacity()
{ {
lsn_t margin; lsn_t margin;
ulint free; ulint free;
bool success = true;
lsn_t smallest_capacity;
log_mutex_enter(); lsn_t smallest_capacity = ((srv_log_file_size_requested
<< srv_page_size_shift)
smallest_capacity = log_group_get_capacity(&log_sys->log); - LOG_FILE_HDR_SIZE)
* srv_n_log_files;
/* Add extra safety */ /* Add extra safety */
smallest_capacity = smallest_capacity - smallest_capacity / 10; smallest_capacity -= smallest_capacity / 10;
/* For each OS thread we must reserve so much free space in the /* For each OS thread we must reserve so much free space in the
smallest log group that it can accommodate the log entries produced smallest log group that it can accommodate the log entries produced
@ -694,15 +693,20 @@ log_set_capacity()
free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency) free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency)
+ LOG_CHECKPOINT_EXTRA_FREE; + LOG_CHECKPOINT_EXTRA_FREE;
if (free >= smallest_capacity / 2) { if (free >= smallest_capacity / 2) {
success = false; ib::error() << "Cannot continue operation. ib_logfiles are too"
" small for innodb_thread_concurrency="
goto failure; << srv_thread_concurrency << ". The combined size of"
} else { " ib_logfiles should be bigger than"
margin = smallest_capacity - free; " 200 kB * innodb_thread_concurrency. "
<< INNODB_PARAMETERS_MSG;
return(false);
} }
margin = smallest_capacity - free;
margin = margin - margin / 10; /* Add still some extra safety */ margin = margin - margin / 10; /* Add still some extra safety */
log_mutex_enter();
log_sys->log_group_capacity = smallest_capacity; log_sys->log_group_capacity = smallest_capacity;
log_sys->max_modified_age_async = margin log_sys->max_modified_age_async = margin
@ -714,19 +718,9 @@ log_set_capacity()
/ LOG_POOL_CHECKPOINT_RATIO_ASYNC; / LOG_POOL_CHECKPOINT_RATIO_ASYNC;
log_sys->max_checkpoint_age = margin; log_sys->max_checkpoint_age = margin;
failure:
log_mutex_exit(); log_mutex_exit();
if (!success) { return(true);
ib::error() << "Cannot continue operation. ib_logfiles are too"
" small for innodb_thread_concurrency="
<< srv_thread_concurrency << ". The combined size of"
" ib_logfiles should be bigger than"
" 200 kB * innodb_thread_concurrency. "
<< INNODB_PARAMETERS_MSG;
}
return(success);
} }
/** Initializes the redo logging subsystem. */ /** Initializes the redo logging subsystem. */

64
storage/innobase/row/row0sel.cc
View File

@ -2231,16 +2231,7 @@ stop_for_a_while:
btr_pcur_store_position(&(plan->pcur), &mtr); btr_pcur_store_position(&(plan->pcur), &mtr);
mtr_commit(&mtr); mtr_commit(&mtr);
ut_ad(!sync_check_iterate(sync_check()));
#ifdef BTR_CUR_HASH_ADAPT
# ifdef UNIV_DEBUG
{
btrsea_sync_check check(true);
ut_ad(!sync_check_iterate(check));
}
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
err = DB_SUCCESS; err = DB_SUCCESS;
goto func_exit; goto func_exit;
@ -2258,14 +2249,7 @@ commit_mtr_for_a_while:
mtr_commit(&mtr); mtr_commit(&mtr);
mtr_has_extra_clust_latch = FALSE; mtr_has_extra_clust_latch = FALSE;
ut_ad(!sync_check_iterate(dict_sync_check()));
#ifdef UNIV_DEBUG
{
dict_sync_check check(true);
ut_ad(!sync_check_iterate(check));
}
#endif /* UNIV_DEBUG */
goto table_loop; goto table_loop;
@ -2280,20 +2264,13 @@ lock_wait_or_error:
mtr_commit(&mtr); mtr_commit(&mtr);
#ifdef UNIV_DEBUG
{
dict_sync_check check(true);
ut_ad(!sync_check_iterate(check));
}
#endif /* UNIV_DEBUG */
func_exit: func_exit:
#ifdef BTR_CUR_HASH_ADAPT #ifdef BTR_CUR_HASH_ADAPT
if (search_latch_locked) { if (search_latch_locked) {
btr_search_s_unlock(index); btr_search_s_unlock(index);
} }
#endif /* BTR_CUR_HASH_ADAPT */ #endif /* BTR_CUR_HASH_ADAPT */
ut_ad(!sync_check_iterate(dict_sync_check()));
if (heap != NULL) { if (heap != NULL) {
mem_heap_free(heap); mem_heap_free(heap);
@ -3041,7 +3018,6 @@ row_sel_store_mysql_field_func(
mem_heap_t* heap; mem_heap_t* heap;
/* Copy an externally stored field to a temporary heap */ /* Copy an externally stored field to a temporary heap */
trx_assert_no_search_latch(prebuilt->trx);
ut_ad(field_no == templ->clust_rec_field_no); ut_ad(field_no == templ->clust_rec_field_no);
ut_ad(templ->type != DATA_POINT); ut_ad(templ->type != DATA_POINT);
@ -3928,11 +3904,7 @@ row_sel_try_search_shortcut_for_mysql(
ut_ad(!prebuilt->templ_contains_blob); ut_ad(!prebuilt->templ_contains_blob);
btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE, btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE,
BTR_SEARCH_LEAF, pcur, BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr);
(trx->has_search_latch)
? RW_S_LATCH
: 0,
mtr);
rec = btr_pcur_get_rec(pcur); rec = btr_pcur_get_rec(pcur);
if (!page_rec_is_user_rec(rec)) { if (!page_rec_is_user_rec(rec)) {
@ -4190,14 +4162,7 @@ row_search_mvcc(
DBUG_RETURN(DB_END_OF_INDEX); DBUG_RETURN(DB_END_OF_INDEX);
} }
#ifdef BTR_CUR_HASH_ADAPT ut_ad(!sync_check_iterate(sync_check()));
# ifdef UNIV_DEBUG
{
btrsea_sync_check check(trx->has_search_latch);
ut_ad(!sync_check_iterate(check));
}
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
if (dict_table_is_discarded(prebuilt->table)) { if (dict_table_is_discarded(prebuilt->table)) {
@ -4222,8 +4187,6 @@ row_search_mvcc(
&& (prebuilt->read_just_key && (prebuilt->read_just_key
|| prebuilt->m_read_virtual_key); || prebuilt->m_read_virtual_key);
trx_assert_no_search_latch(trx);
/* Reset the new record lock info if srv_locks_unsafe_for_binlog /* Reset the new record lock info if srv_locks_unsafe_for_binlog
is set or session is using a READ COMMITED isolation level. Then is set or session is using a READ COMMITED isolation level. Then
we are able to remove the record locks set here on an individual we are able to remove the record locks set here on an individual
@ -4378,9 +4341,7 @@ row_search_mvcc(
and if we try that, we can deadlock on the adaptive and if we try that, we can deadlock on the adaptive
hash index semaphore! */ hash index semaphore! */
trx_assert_no_search_latch(trx);
rw_lock_s_lock(btr_get_search_latch(index)); rw_lock_s_lock(btr_get_search_latch(index));
trx->has_search_latch = true;
switch (row_sel_try_search_shortcut_for_mysql( switch (row_sel_try_search_shortcut_for_mysql(
&rec, prebuilt, &offsets, &heap, &rec, prebuilt, &offsets, &heap,
@ -4435,7 +4396,6 @@ row_search_mvcc(
err = DB_SUCCESS; err = DB_SUCCESS;
rw_lock_s_unlock(btr_get_search_latch(index)); rw_lock_s_unlock(btr_get_search_latch(index));
trx->has_search_latch = false;
goto func_exit; goto func_exit;
@ -4446,7 +4406,6 @@ row_search_mvcc(
err = DB_RECORD_NOT_FOUND; err = DB_RECORD_NOT_FOUND;
rw_lock_s_unlock(btr_get_search_latch(index)); rw_lock_s_unlock(btr_get_search_latch(index));
trx->has_search_latch = false;
/* NOTE that we do NOT store the cursor /* NOTE that we do NOT store the cursor
position */ position */
@ -4464,7 +4423,6 @@ row_search_mvcc(
mtr_start(&mtr); mtr_start(&mtr);
rw_lock_s_unlock(btr_get_search_latch(index)); rw_lock_s_unlock(btr_get_search_latch(index));
trx->has_search_latch = false;
} }
} }
#endif /* BTR_CUR_HASH_ADAPT */ #endif /* BTR_CUR_HASH_ADAPT */
@ -4472,8 +4430,6 @@ row_search_mvcc(
/*-------------------------------------------------------------*/ /*-------------------------------------------------------------*/
/* PHASE 3: Open or restore index cursor position */ /* PHASE 3: Open or restore index cursor position */
trx_assert_no_search_latch(trx);
spatial_search = dict_index_is_spatial(index) spatial_search = dict_index_is_spatial(index)
&& mode >= PAGE_CUR_CONTAIN; && mode >= PAGE_CUR_CONTAIN;
@ -5787,15 +5743,7 @@ func_exit:
} }
} }
#ifdef BTR_CUR_HASH_ADAPT ut_ad(!sync_check_iterate(sync_check()));
# ifdef UNIV_DEBUG
{
btrsea_sync_check check(trx->has_search_latch);
ut_ad(!sync_check_iterate(check));
}
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
DEBUG_SYNC_C("innodb_row_search_for_mysql_exit"); DEBUG_SYNC_C("innodb_row_search_for_mysql_exit");

35
storage/innobase/srv/srv0conc.cc
View File

@ -197,11 +197,6 @@ srv_conc_enter_innodb_with_atomics(
(void) my_atomic_addlint( (void) my_atomic_addlint(
&srv_conc.n_waiting, 1); &srv_conc.n_waiting, 1);
/* Release possible search system latch this
thread has */
trx_assert_no_search_latch(trx);
thd_wait_begin(trx->mysql_thd, THD_WAIT_USER_LOCK); thd_wait_begin(trx->mysql_thd, THD_WAIT_USER_LOCK);
notified_mysql = TRUE; notified_mysql = TRUE;
@ -257,15 +252,7 @@ srv_conc_enter_innodb(
{ {
trx_t* trx = prebuilt->trx; trx_t* trx = prebuilt->trx;
#ifdef BTR_CUR_HASH_ADAPT ut_ad(!sync_check_iterate(sync_check()));
# ifdef UNIV_DEBUG
{
btrsea_sync_check check(trx->has_search_latch);
ut_ad(!sync_check_iterate(check));
}
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
srv_conc_enter_innodb_with_atomics(trx); srv_conc_enter_innodb_with_atomics(trx);
} }
@ -279,15 +266,7 @@ srv_conc_force_enter_innodb(
trx_t* trx) /*!< in: transaction object associated with the trx_t* trx) /*!< in: transaction object associated with the
thread */ thread */
{ {
#ifdef BTR_CUR_HASH_ADAPT ut_ad(!sync_check_iterate(sync_check()));
# ifdef UNIV_DEBUG
{
btrsea_sync_check check(trx->has_search_latch);
ut_ad(!sync_check_iterate(check));
}
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
if (!srv_thread_concurrency) { if (!srv_thread_concurrency) {
@ -320,15 +299,7 @@ srv_conc_force_exit_innodb(
srv_conc_exit_innodb_with_atomics(trx); srv_conc_exit_innodb_with_atomics(trx);
#ifdef BTR_CUR_HASH_ADAPT ut_ad(!sync_check_iterate(sync_check()));
# ifdef UNIV_DEBUG
{
btrsea_sync_check check(trx->has_search_latch);
ut_ad(!sync_check_iterate(check));
}
# endif /* UNIV_DEBUG */
#endif /* BTR_CUR_HASH_ADAPT */
} }
/*********************************************************************//** /*********************************************************************//**

37
storage/innobase/sync/sync0debug.cc
View File

@ -1,7 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. Copyright (c) 2017, MariaDB Corporation.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described Google, Inc. Those modifications are gratefully acknowledged and are described
@ -298,29 +298,23 @@ struct LatchDebug {
} }
/** Iterate over a thread's latches. /** Iterate over a thread's latches.
@param[in,out] functor The callback @param[in] functor The callback
@return true if the functor returns true. */ @return true if the functor returns true. */
bool for_each(sync_check_functor_t& functor) bool for_each(const sync_check_functor_t& functor)
UNIV_NOTHROW UNIV_NOTHROW
{ {
const Latches* latches = thread_latches(); if (const Latches* latches = thread_latches()) {
Latches::const_iterator end = latches->end();
for (Latches::const_iterator it = latches->begin();
it != end; ++it) {
if (latches == 0) { if (functor(it->m_level)) {
return(functor.result()); return(true);
} }
Latches::const_iterator end = latches->end();
for (Latches::const_iterator it = latches->begin();
it != end;
++it) {
if (functor(it->m_level)) {
break;
} }
} }
return(functor.result()); return(false);
} }
/** Removes a latch from the thread level array if it is found there. /** Removes a latch from the thread level array if it is found there.
@ -1215,13 +1209,12 @@ sync_check_find(latch_level_t level)
/** Iterate over the thread's latches. /** Iterate over the thread's latches.
@param[in,out] functor called for each element. @param[in,out] functor called for each element.
@return false if the sync debug hasn't been initialised @return true if the functor returns true for any element */
@return the value returned by the functor */
bool bool
sync_check_iterate(sync_check_functor_t& functor) sync_check_iterate(const sync_check_functor_t& functor)
{ {
if (LatchDebug::instance() != NULL) { if (LatchDebug* debug = LatchDebug::instance()) {
return(LatchDebug::instance()->for_each(functor)); return(debug->for_each(functor));
} }
return(false); return(false);

4
storage/innobase/trx/trx0i_s.cc
View File

@ -589,10 +589,6 @@ thd_done:
row->trx_foreign_key_error = NULL; row->trx_foreign_key_error = NULL;
} }
#ifdef BTR_CUR_HASH_ADAPT
row->trx_has_search_latch = (ibool) trx->has_search_latch;
#endif /* BTR_CUR_HASH_ADAPT */
row->trx_is_read_only = trx->read_only; row->trx_is_read_only = trx->read_only;
row->trx_is_autocommit_non_locking = trx_is_autocommit_non_locking(trx); row->trx_is_autocommit_non_locking = trx_is_autocommit_non_locking(trx);

17
storage/innobase/trx/trx0trx.cc
View File

@ -272,8 +272,6 @@ struct TrxFactory {
ut_a(trx->lock.wait_lock == NULL); ut_a(trx->lock.wait_lock == NULL);
ut_a(trx->lock.wait_thr == NULL); ut_a(trx->lock.wait_thr == NULL);
trx_assert_no_search_latch(trx);
ut_a(trx->dict_operation_lock_mode == 0); ut_a(trx->dict_operation_lock_mode == 0);
if (trx->lock.lock_heap != NULL) { if (trx->lock.lock_heap != NULL) {
@ -341,9 +339,6 @@ struct TrxFactory {
ut_a(trx->lock.wait_thr == NULL); ut_a(trx->lock.wait_thr == NULL);
ut_a(trx->lock.wait_lock == NULL); ut_a(trx->lock.wait_lock == NULL);
trx_assert_no_search_latch(trx);
ut_a(trx->dict_operation_lock_mode == 0); ut_a(trx->dict_operation_lock_mode == 0);
ut_a(UT_LIST_GET_LEN(trx->lock.trx_locks) == 0); ut_a(UT_LIST_GET_LEN(trx->lock.trx_locks) == 0);
@ -2413,13 +2408,6 @@ state_ok:
(ulong) n_rec_locks); (ulong) n_rec_locks);
} }
#ifdef BTR_CUR_HASH_ADAPT
if (trx->has_search_latch) {
newline = TRUE;
fputs(", holds adaptive hash latch", f);
}
#endif /* BTR_CUR_HASH_ADAPT */
if (trx->undo_no != 0) { if (trx->undo_no != 0) {
newline = TRUE; newline = TRUE;
fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no); fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no);
@ -2551,11 +2539,6 @@ state_ok:
fprintf(f, "que state %lu ", (ulong) trx->lock.que_state); fprintf(f, "que state %lu ", (ulong) trx->lock.que_state);
} }
if (trx->has_search_latch) {
newline = TRUE;
fputs(", holds adaptive hash latch", f);
}
if (trx->undo_no != 0) { if (trx->undo_no != 0) {
newline = TRUE; newline = TRUE;
fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no); fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no);

3
storage/myisam/ha_myisam.cc
View File

@ -1183,9 +1183,6 @@ int ha_myisam::repair(THD *thd, HA_CHECK &param, bool do_optimize)
share->state.dupp_key= MI_MAX_KEY; share->state.dupp_key= MI_MAX_KEY;
strmov(fixed_name,file->filename); strmov(fixed_name,file->filename);
// Release latches since this can take a long time
ha_release_temporary_latches(thd);
/* /*
Don't lock tables if we have used LOCK TABLE or if we come from Don't lock tables if we have used LOCK TABLE or if we come from
enable_index() enable_index()

42
storage/xtradb/handler/ha_innodb.cc
View File

@ -2122,39 +2122,6 @@ ha_innobase::is_fake_change_enabled(THD* thd)
return(trx && UNIV_UNLIKELY(trx->fake_changes)); return(trx && UNIV_UNLIKELY(trx->fake_changes));
} }
/********************************************************************//**
In XtraDB it is impossible for a transaction to own a search latch outside of
InnoDB code, so there is nothing to release on demand. We keep this function to
simplify maintenance.
@return 0 */
static
int
innobase_release_temporary_latches(
/*===============================*/
handlerton* hton MY_ATTRIBUTE((unused)), /*!< in: handlerton */
THD* thd MY_ATTRIBUTE((unused))) /*!< in: MySQL thread */
{
#ifdef UNIV_DEBUG
DBUG_ASSERT(hton == innodb_hton_ptr);
if (!innodb_inited || thd == NULL) {
return(0);
}
trx_t* trx = thd_to_trx(thd);
if (trx != NULL) {
#ifdef UNIV_SYNC_DEBUG
ut_ad(!btr_search_own_any());
#endif
trx_search_latch_release_if_reserved(trx);
}
#endif
return(0);
}
/********************************************************************//** /********************************************************************//**
Increments innobase_active_counter and every INNOBASE_WAKE_INTERVALth Increments innobase_active_counter and every INNOBASE_WAKE_INTERVALth
time calls srv_active_wake_master_thread. This function should be used time calls srv_active_wake_master_thread. This function should be used
@ -3813,9 +3780,6 @@ innobase_init(
innobase_hton->flags = HTON_SUPPORTS_EXTENDED_KEYS | innobase_hton->flags = HTON_SUPPORTS_EXTENDED_KEYS |
HTON_SUPPORTS_FOREIGN_KEYS; HTON_SUPPORTS_FOREIGN_KEYS;
innobase_hton->release_temporary_latches =
innobase_release_temporary_latches;
innobase_hton->kill_query = innobase_kill_connection; innobase_hton->kill_query = innobase_kill_connection;
if (srv_file_per_table) if (srv_file_per_table)
@ -6262,9 +6226,6 @@ ha_innobase::open(
thd = ha_thd(); thd = ha_thd();
/* No-op in XtraDB */
innobase_release_temporary_latches(ht, thd);
normalize_table_name(norm_name, name); normalize_table_name(norm_name, name);
user_thd = NULL; user_thd = NULL;
@ -6724,9 +6685,6 @@ ha_innobase::close()
thd = ha_thd(); thd = ha_thd();
/* No-op in XtraDB */
innobase_release_temporary_latches(ht, thd);
row_prebuilt_free(prebuilt, FALSE); row_prebuilt_free(prebuilt, FALSE);
if (upd_buf != NULL) { if (upd_buf != NULL) {

71
storage/xtradb/log/log0log.cc
View File

@ -860,43 +860,16 @@ ibool
log_calc_max_ages(void) log_calc_max_ages(void)
/*===================*/ /*===================*/
{ {
log_group_t* group;
lsn_t margin; lsn_t margin;
ulint free; ulint free;
ibool success = TRUE;
lsn_t smallest_capacity;
lsn_t archive_margin;
lsn_t smallest_archive_margin;
mutex_enter(&(log_sys->mutex)); lsn_t smallest_capacity = ((srv_log_file_size_requested
<< srv_page_size_shift)
group = UT_LIST_GET_FIRST(log_sys->log_groups); - LOG_FILE_HDR_SIZE)
* srv_n_log_files;
ut_ad(group);
smallest_capacity = LSN_MAX;
smallest_archive_margin = LSN_MAX;
while (group) {
if (log_group_get_capacity(group) < smallest_capacity) {
smallest_capacity = log_group_get_capacity(group);
}
archive_margin = log_group_get_capacity(group)
- (group->file_size - LOG_FILE_HDR_SIZE)
- LOG_ARCHIVE_EXTRA_MARGIN;
if (archive_margin < smallest_archive_margin) {
smallest_archive_margin = archive_margin;
}
group = UT_LIST_GET_NEXT(log_groups, group);
}
/* Add extra safety */ /* Add extra safety */
smallest_capacity = smallest_capacity - smallest_capacity / 10; smallest_capacity -= smallest_capacity / 10;
/* For each OS thread we must reserve so much free space in the /* For each OS thread we must reserve so much free space in the
smallest log group that it can accommodate the log entries produced smallest log group that it can accommodate the log entries produced
@ -906,15 +879,16 @@ log_calc_max_ages(void)
free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency) free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency)
+ LOG_CHECKPOINT_EXTRA_FREE; + LOG_CHECKPOINT_EXTRA_FREE;
if (free >= smallest_capacity / 2) { if (free >= smallest_capacity / 2) {
success = FALSE; ib_logf(IB_LOG_LEVEL_FATAL,
"The combined size of ib_logfiles"
goto failure; " should be bigger than\n"
} else { "InnoDB: 200 kB * innodb_thread_concurrency.");
margin = smallest_capacity - free;
} }
margin = smallest_capacity - free;
margin = margin - margin / 10; /* Add still some extra safety */ margin = margin - margin / 10; /* Add still some extra safety */
mutex_enter(&log_sys->mutex);
log_sys->log_group_capacity = smallest_capacity; log_sys->log_group_capacity = smallest_capacity;
log_sys->max_modified_age_async = margin log_sys->max_modified_age_async = margin
@ -927,22 +901,17 @@ log_calc_max_ages(void)
log_sys->max_checkpoint_age = margin; log_sys->max_checkpoint_age = margin;
#ifdef UNIV_LOG_ARCHIVE #ifdef UNIV_LOG_ARCHIVE
log_sys->max_archived_lsn_age = smallest_archive_margin; lsn_t archive_margin = smallest_capacity
- (srv_log_file_size_requested - LOG_FILE_HDR_SIZE)
- LOG_ARCHIVE_EXTRA_MARGIN;
log_sys->max_archived_lsn_age = archive_margin;
log_sys->max_archived_lsn_age_async = smallest_archive_margin log_sys->max_archived_lsn_age_async = archive_margin
- smallest_archive_margin / LOG_ARCHIVE_RATIO_ASYNC; - archive_margin / LOG_ARCHIVE_RATIO_ASYNC;
#endif /* UNIV_LOG_ARCHIVE */ #endif /* UNIV_LOG_ARCHIVE */
failure: mutex_exit(&log_sys->mutex);
mutex_exit(&(log_sys->mutex));
if (!success) { return(true);
ib_logf(IB_LOG_LEVEL_FATAL,
"The combined size of ib_logfiles"
" should be bigger than\n"
"InnoDB: 200 kB * innodb_thread_concurrency.");
}
return(success);
} }
/******************************************************//** /******************************************************//**