From a79d9bff8a72066c208d86faf26d8916635ef67d Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Mon, 26 Jan 2015 10:39:23 -0500 Subject: [PATCH 01/10] update tokudb README for 7.5.5 --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2ab2e21a5a1..0d4f09d4c86 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,14 @@ working MySQL or MariaDB with Tokutek patches, and with the TokuDB storage engine, called `make.mysql.bash`. This script will download copies of the needed source code from github and build everything. -To build MySQL 5.5.40 with TokuDB 7.5.3: +To build MySQL 5.5.41 with TokuDB 7.5.5: ```sh -scripts/make.mysql.bash --mysqlbuild=mysql-5.5.40-tokudb-7.5.3-linux-x86_64 +scripts/make.mysql.bash --mysqlbuild=mysql-5.5.41-tokudb-7.5.5-linux-x86_64 ``` -To build MariaDB 5.5.40 with TokuDB 7.5.3: +To build MariaDB 5.5.41 with TokuDB 7.5.5: ```sh -scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.40-tokudb-7.5.3-linux-x86_64 +scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.41-tokudb-7.5.5-linux-x86_64 ``` Before you start, make sure you have a C++11-compatible compiler (GCC >= @@ -59,6 +59,7 @@ repositories, run this: scripts/make.mysql.debug.env.bash ``` +We use gcc from devtoolset-1.1 on CentOS 5.9 for builds. Contribute ---------- From bc605380b11b324f8f9714e1109c9e983b1c68e2 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Thu, 29 Jan 2015 08:56:01 -0500 Subject: [PATCH 02/10] DB-803 fix the 5733 test to be more tolerant of join type --- mysql-test/suite/tokudb.bugs/r/5733_tokudb.result | 2 +- mysql-test/suite/tokudb.bugs/t/5733_tokudb.test | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/tokudb.bugs/r/5733_tokudb.result b/mysql-test/suite/tokudb.bugs/r/5733_tokudb.result index b642c8d6963..1db2c5746e2 100644 --- a/mysql-test/suite/tokudb.bugs/r/5733_tokudb.result +++ b/mysql-test/suite/tokudb.bugs/r/5733_tokudb.result @@ -10003,7 +10003,7 @@ insert into t values (9998,0); insert into t values (9999,0); explain select id from t where id>0 limit 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t range PRIMARY PRIMARY 8 NULL # Using where; Using index +1 SIMPLE t range_or_index PRIMARY PRIMARY 8 NULL # Using where; Using index explain select * from t where id>0 limit 10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t range PRIMARY PRIMARY 8 NULL # Using where diff --git a/mysql-test/suite/tokudb.bugs/t/5733_tokudb.test b/mysql-test/suite/tokudb.bugs/t/5733_tokudb.test index e0e0fc886fd..659caafd5ac 100644 --- a/mysql-test/suite/tokudb.bugs/t/5733_tokudb.test +++ b/mysql-test/suite/tokudb.bugs/t/5733_tokudb.test @@ -18,7 +18,18 @@ while ($i < $n) { inc $i; } -replace_column 9 #; +# the plan for the following query should be a range scan. about 1 of 10 times, +# the plan is an index scan. the different scan type occurs because the query optimizer +# is handed different row counts by tokudb::records_in_range. the cost estimates made +# by the query optimizer are very close to begin with. sometimes, the cost of an index +# scan is less than the cost of a range scan. +# +# if a tokudb checkpoint occurs before this query is run, then the records_in_range +# function returns a larger than expected row estimate. +# +# column 4 is the join type (should be range or index) +# column 9 is the estimated key count +replace_column 4 range_or_index 9 #; explain select id from t where id>0 limit 10; replace_column 9 #; From 67ad13e73963e2801e9369f512f182d891f4fb2c Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Thu, 29 Jan 2015 09:00:42 -0500 Subject: [PATCH 03/10] DB-793 add debug return prints for scan_time, read_time, key_read_time, and index_only_read_time to help debug query plan issues --- storage/tokudb/ha_tokudb.cc | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index 7c9d295ba59..c7b3302c248 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -7154,12 +7154,15 @@ To rename the table, make sure no transactions touch the table.", from, to); double ha_tokudb::scan_time() { TOKUDB_HANDLER_DBUG_ENTER(""); double ret_val = (double)stats.records / 3; + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %" PRIu64 " %f", (uint64_t) stats.records, ret_val); + } DBUG_RETURN(ret_val); } double ha_tokudb::keyread_time(uint index, uint ranges, ha_rows rows) { - TOKUDB_HANDLER_DBUG_ENTER(""); + TOKUDB_HANDLER_DBUG_ENTER("%u %u %" PRIu64, index, ranges, (uint64_t) rows); double ret_val; if (index == primary_key || key_is_clustering(&table->key_info[index])) { ret_val = read_time(index, ranges, rows); @@ -7177,6 +7180,9 @@ double ha_tokudb::keyread_time(uint index, uint ranges, ha_rows rows) (table->key_info[index].key_length + ref_length) + 1); ret_val = (rows + keys_per_block - 1)/ keys_per_block; + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %f", ret_val); + } DBUG_RETURN(ret_val); } @@ -7197,7 +7203,7 @@ double ha_tokudb::read_time( ha_rows rows ) { - TOKUDB_HANDLER_DBUG_ENTER(""); + TOKUDB_HANDLER_DBUG_ENTER("%u %u %" PRIu64, index, ranges, (uint64_t) rows); double total_scan; double ret_val; bool is_primary = (index == primary_key); @@ -7239,12 +7245,18 @@ double ha_tokudb::read_time( ret_val = is_clustering ? ret_val + 0.00001 : ret_val; cleanup: + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %f", ret_val); + } DBUG_RETURN(ret_val); } double ha_tokudb::index_only_read_time(uint keynr, double records) { - TOKUDB_HANDLER_DBUG_ENTER(""); + TOKUDB_HANDLER_DBUG_ENTER("%u %f", keynr, records); double ret_val = keyread_time(keynr, 1, (ha_rows)records); + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %f", ret_val); + } DBUG_RETURN(ret_val); } @@ -7319,7 +7331,7 @@ ha_rows ha_tokudb::records_in_range(uint keynr, key_range* start_key, key_range* cleanup: if (tokudb_debug & TOKUDB_DEBUG_RETURN) { - TOKUDB_HANDLER_TRACE("%" PRIu64 " %" PRIu64, (uint64_t) ret_val, rows); + TOKUDB_HANDLER_TRACE("return %" PRIu64 " %" PRIu64, (uint64_t) ret_val, rows); } DBUG_RETURN(ret_val); } From 54fd89633035f604951dbbfcd95ef6a1bbe35f57 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Mon, 2 Feb 2015 08:50:43 -0500 Subject: [PATCH 04/10] DBP-13 build standalone backup plugin --- scripts/make.mysql.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/make.mysql.bash b/scripts/make.mysql.bash index a614424d9a0..c1259797590 100755 --- a/scripts/make.mysql.bash +++ b/scripts/make.mysql.bash @@ -117,6 +117,7 @@ elif [ $build_type = enterprise ] ; then github_download Tokutek/tokudb-backup-plugin $(git_tree $git_tag $backup_tree) tokudb-backup-plugin mv tokudb-backup-plugin plugin github_download Tokutek/backup-enterprise $(git_tree $git_tag $backup_tree) backup-enterprise + rm -rf plugin/tokudb-backup-plugin/backup mv backup-enterprise/backup plugin/tokudb-backup-plugin rm -rf backup-enterprise fi From a06c7d6a2ad032b671a080e33aafa9b9970deebf Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Mon, 2 Feb 2015 18:15:37 -0500 Subject: [PATCH 05/10] DB-808 skip sync of the log if tokudb_fsync_log_period > 0 --- storage/tokudb/hatoku_hton.cc | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index f6509794546..233b6023d3f 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -777,7 +777,7 @@ extern "C" enum durability_properties thd_get_durability_property(const MYSQL_TH #endif // Determine if an fsync is used when a transaction is committed. -static bool tokudb_fsync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) { +static bool tokudb_sync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) { #if MYSQL_VERSION_ID >= 50600 // Check the client durability property which is set during 2PC if (thd_get_durability_property(thd) == HA_IGNORE_DURABILITY) @@ -788,6 +788,8 @@ static bool tokudb_fsync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) if (txn->is_prepared(txn) && mysql_bin_log.is_open()) return false; #endif + if (tokudb_fsync_log_period > 0) + return false; return THDVAR(thd, commit_sync) != 0; } @@ -798,7 +800,7 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { DB_TXN **txn = all ? &trx->all : &trx->stmt; DB_TXN *this_txn = *txn; if (this_txn) { - uint32_t syncflag = tokudb_fsync_on_commit(thd, trx, this_txn) ? 0 : DB_TXN_NOSYNC; + uint32_t syncflag = tokudb_sync_on_commit(thd, trx, this_txn) ? 0 : DB_TXN_NOSYNC; if (tokudb_debug & TOKUDB_DEBUG_TXN) { TOKUDB_TRACE("commit trx %u txn %p syncflag %u", all, this_txn, syncflag); } @@ -849,6 +851,13 @@ static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { } #if TOKU_INCLUDE_XA +static bool tokudb_sync_on_prepare(void) { + // skip sync of log if fsync log period > 0 + if (tokudb_fsync_log_period > 0) + return false; + else + return true; +} static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { TOKUDB_DBUG_ENTER(""); @@ -863,6 +872,7 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN* txn = all ? trx->all : trx->stmt; if (txn) { + uint32_t syncflag = tokudb_sync_on_prepare() ? 0 : DB_TXN_NOSYNC; if (tokudb_debug & TOKUDB_DEBUG_TXN) { TOKUDB_TRACE("doing txn prepare:%d:%p", all, txn); } @@ -871,7 +881,7 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { thd_get_xid(thd, (MYSQL_XID*) &thd_xid); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_prepare_before", DBUG_SUICIDE();); - r = txn->xa_prepare(txn, &thd_xid); + r = txn->xa_prepare(txn, &thd_xid, syncflag); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_prepare_after", DBUG_SUICIDE();); } From e28727b562d35ed04c29345300a070043f07d390 Mon Sep 17 00:00:00 2001 From: Rik Prohaska Date: Sat, 7 Feb 2015 17:11:49 -0500 Subject: [PATCH 06/10] build tokudb debug env with backup plugin --- scripts/make.mysql.debug.env.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/make.mysql.debug.env.bash b/scripts/make.mysql.debug.env.bash index 08ea19827bc..3f8b4e6c219 100755 --- a/scripts/make.mysql.debug.env.bash +++ b/scripts/make.mysql.debug.env.bash @@ -62,7 +62,7 @@ tokudbengine=tokudb-engine tokudbengine_tree=master ftindex=ft-index ftindex_tree=master -backup=backup-community +backup=tokudb-backup-plugin backup_tree=master cc=gcc cxx=g++ @@ -119,9 +119,9 @@ if [ $? != 0 ] ; then exit 1; fi ln -s ../../$tokudbengine/storage/tokudb tokudb if [ $? != 0 ] ; then exit 1; fi popd -pushd $mysql_tree +pushd $mysql_tree/plugin if [ $? != 0 ] ; then exit 1; fi -ln -s ../$backup/backup toku_backup +ln -s ../../$backup $backup if [ $? != 0 ] ; then exit 1; fi popd pushd $mysql_tree/scripts From bd026f29f9d68e5b3ed474c8eba05a89f16d44d1 Mon Sep 17 00:00:00 2001 From: Rik Prohaska Date: Mon, 16 Feb 2015 10:46:46 -0500 Subject: [PATCH 07/10] DB-813 add an interrupt callback to the analyze cursor to allow early termination, plus detect garbage rows during analyze scan --- storage/tokudb/ha_tokudb_admin.cc | 47 +++++++++++++++++++++++++------ storage/tokudb/hatoku_hton.cc | 1 + storage/tokudb/hatoku_hton.h | 15 +++------- storage/tokudb/tokudb_card.h | 35 ++++++++++++++++++----- 4 files changed, 71 insertions(+), 27 deletions(-) diff --git a/storage/tokudb/ha_tokudb_admin.cc b/storage/tokudb/ha_tokudb_admin.cc index d6da45733a5..b109cd1b976 100644 --- a/storage/tokudb/ha_tokudb_admin.cc +++ b/storage/tokudb/ha_tokudb_admin.cc @@ -156,18 +156,47 @@ int ha_tokudb::analyze(THD *thd, HA_CHECK_OPT *check_opt) { bool is_unique = false; if (i == primary_key || (key_info->flags & HA_NOSAME)) is_unique = true; + uint64_t rows = 0; + uint64_t deleted_rows = 0; int error = tokudb::analyze_card(share->key_file[i], txn, is_unique, num_key_parts, &rec_per_key[total_key_parts], - tokudb_cmp_dbt_key_parts, analyze_progress, &analyze_progress_extra); + tokudb_cmp_dbt_key_parts, analyze_progress, &analyze_progress_extra, + &rows, &deleted_rows); + sql_print_information("tokudb analyze %d %" PRIu64 " %" PRIu64, error, rows, deleted_rows); if (error != 0 && error != ETIME) { result = HA_ADMIN_FAILED; - } else { - // debug - if (tokudb_debug & TOKUDB_DEBUG_ANALYZE) { - TOKUDB_HANDLER_TRACE("%s.%s.%s", - table_share->db.str, table_share->table_name.str, i == primary_key ? "primary" : table_share->key_info[i].name); - for (uint j = 0; j < num_key_parts; j++) - TOKUDB_HANDLER_TRACE("%lu", rec_per_key[total_key_parts+j]); - } + } + if (error != 0 && rows == 0 && deleted_rows > 0) { + result = HA_ADMIN_FAILED; + } + double f = THDVAR(thd, analyze_delete_fraction); + if (result == HA_ADMIN_FAILED || (double) deleted_rows > f * (rows + deleted_rows)) { + char name[256]; int namelen; + namelen = snprintf(name, sizeof name, "%.*s.%.*s.%s", + (int) table_share->db.length, table_share->db.str, + (int) table_share->table_name.length, table_share->table_name.str, + key_name); + thd->protocol->prepare_for_resend(); + thd->protocol->store(name, namelen, system_charset_info); + thd->protocol->store("analyze", 7, system_charset_info); + thd->protocol->store("info", 4, system_charset_info); + char rowmsg[256]; int rowmsglen; + rowmsglen = snprintf(rowmsg, sizeof rowmsg, "rows processed %" PRIu64 " rows deleted %" PRIu64, rows, deleted_rows); + thd->protocol->store(rowmsg, rowmsglen, system_charset_info); + thd->protocol->write(); + + sql_print_information("tokudb analyze on %.*s %.*s", + namelen, name, rowmsglen, rowmsg); + } + if (tokudb_debug & TOKUDB_DEBUG_ANALYZE) { + char name[256]; int namelen; + namelen = snprintf(name, sizeof name, "%.*s.%.*s.%s", + (int) table_share->db.length, table_share->db.str, + (int) table_share->table_name.length, table_share->table_name.str, + key_name); + TOKUDB_HANDLER_TRACE("%.*s rows %" PRIu64 " deleted %" PRIu64, + namelen, name, rows, deleted_rows); + for (uint j = 0; j < num_key_parts; j++) + TOKUDB_HANDLER_TRACE("%lu", rec_per_key[total_key_parts+j]); } total_key_parts += num_key_parts; } diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index 233b6023d3f..bd70c9c0a66 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -1454,6 +1454,7 @@ static struct st_mysql_sys_var *tokudb_system_variables[] = { MYSQL_SYSVAR(disable_slow_upsert), #endif MYSQL_SYSVAR(analyze_time), + MYSQL_SYSVAR(analyze_delete_fraction), MYSQL_SYSVAR(fsync_log_period), #if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL MYSQL_SYSVAR(gdb_path), diff --git a/storage/tokudb/hatoku_hton.h b/storage/tokudb/hatoku_hton.h index bee3598bea3..43d6de241c2 100644 --- a/storage/tokudb/hatoku_hton.h +++ b/storage/tokudb/hatoku_hton.h @@ -370,16 +370,9 @@ static MYSQL_THDVAR_BOOL(disable_slow_upsert, ); #endif -static MYSQL_THDVAR_UINT(analyze_time, - 0, - "analyze time", - NULL, - NULL, - 5, // default - 0, // min - ~0U, // max - 1 // blocksize -); +static MYSQL_THDVAR_UINT(analyze_time, 0, "analyze time (seconds)", NULL /*check*/, NULL /*update*/, 5 /*default*/, 0 /*min*/, ~0U /*max*/, 1 /*blocksize*/); + +static MYSQL_THDVAR_DOUBLE(analyze_delete_fraction, 0, "fraction of rows allowed to be deleted", NULL /*check*/, NULL /*update*/, 1.0 /*def*/, 0 /*min*/, 1.0 /*max*/, 1); static void tokudb_checkpoint_lock(THD * thd); static void tokudb_checkpoint_unlock(THD * thd); @@ -484,7 +477,7 @@ static int tokudb_killed_callback(void) { return thd_killed(thd); } -static bool tokudb_killed_thd_callback(void *extra) { +static bool tokudb_killed_thd_callback(void *extra, uint64_t deleted_rows) { THD *thd = static_cast(extra); return thd_killed(thd) != 0; } diff --git a/storage/tokudb/tokudb_card.h b/storage/tokudb/tokudb_card.h index 797c705bbaf..22e6fb9b3da 100644 --- a/storage/tokudb/tokudb_card.h +++ b/storage/tokudb/tokudb_card.h @@ -218,15 +218,32 @@ namespace tokudb { return error; } + struct analyze_card_cursor_callback_extra { + int (*analyze_progress)(void *extra, uint64_t rows); + void *analyze_extra; + uint64_t *rows; + uint64_t *deleted_rows; + }; + + bool analyze_card_cursor_callback(void *extra, uint64_t deleted_rows) { + analyze_card_cursor_callback_extra *a_extra = static_cast(extra); + *a_extra->deleted_rows += deleted_rows; + int r = a_extra->analyze_progress(a_extra->analyze_extra, *a_extra->rows); + sql_print_information("tokudb analyze_card_cursor_callback %u %" PRIu64 " %" PRIu64, r, *a_extra->deleted_rows, deleted_rows); + return r != 0; + } + // Compute records per key for all key parts of the ith key of the table. // For each key part, put records per key part in *rec_per_key_part[key_part_index]. // Returns 0 if success, otherwise an error number. // TODO statistical dives into the FT int analyze_card(DB *db, DB_TXN *txn, bool is_unique, uint64_t num_key_parts, uint64_t *rec_per_key_part, int (*key_compare)(DB *, const DBT *, const DBT *, uint), - int (*analyze_progress)(void *extra, uint64_t rows), void *progress_extra) { + int (*analyze_progress)(void *extra, uint64_t rows), void *progress_extra, + uint64_t *return_rows, uint64_t *return_deleted_rows) { int error = 0; uint64_t rows = 0; + uint64_t deleted_rows = 0; uint64_t unique_rows[num_key_parts]; if (is_unique && num_key_parts == 1) { // dont compute for unique keys with a single part. we already know the answer. @@ -235,6 +252,8 @@ namespace tokudb { DBC *cursor = NULL; error = db->cursor(db, txn, &cursor, 0); if (error == 0) { + analyze_card_cursor_callback_extra e = { analyze_progress, progress_extra, &rows, &deleted_rows }; + cursor->c_set_check_interrupt_callback(cursor, analyze_card_cursor_callback, &e); for (uint64_t i = 0; i < num_key_parts; i++) unique_rows[i] = 1; // stop looking when the entire dictionary was analyzed, or a cap on execution time was reached, or the analyze was killed. @@ -243,8 +262,8 @@ namespace tokudb { while (1) { error = cursor->c_get(cursor, &key, 0, DB_NEXT); if (error != 0) { - if (error == DB_NOTFOUND) - error = 0; // eof is not an error + if (error == DB_NOTFOUND || error == TOKUDB_INTERRUPTED) + error = 0; // not an error break; } rows++; @@ -287,10 +306,12 @@ namespace tokudb { } } // return cardinality - if (error == 0 || error == ETIME) { - for (uint64_t i = 0; i < num_key_parts; i++) - rec_per_key_part[i] = rows / unique_rows[i]; - } + if (return_rows) + *return_rows = rows; + if (return_deleted_rows) + *return_deleted_rows = deleted_rows; + for (uint64_t i = 0; i < num_key_parts; i++) + rec_per_key_part[i] = rows / unique_rows[i]; return error; } } From 084668af6be738f8bc26ea4015229242ee04bdc1 Mon Sep 17 00:00:00 2001 From: Rik Prohaska Date: Wed, 18 Feb 2015 01:46:09 -0500 Subject: [PATCH 08/10] DB-814 cleanup trx->sp_level when master txn (trx->all) is retired --- storage/tokudb/hatoku_hton.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index bd70c9c0a66..23b3fd028ed 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -794,7 +794,7 @@ static bool tokudb_sync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) { } static int tokudb_commit(handlerton * hton, THD * thd, bool all) { - TOKUDB_DBUG_ENTER(""); + TOKUDB_DBUG_ENTER("%u", all); DBUG_PRINT("trans", ("ending transaction %s", all ? "all" : "stmt")); tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN **txn = all ? &trx->all : &trx->stmt; @@ -810,11 +810,11 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { commit_txn_with_progress(this_txn, syncflag, thd); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_commit_after", DBUG_SUICIDE();); - if (this_txn == trx->sp_level) { - trx->sp_level = 0; - } - *txn = 0; + *txn = NULL; trx->sub_sp_level = NULL; + if (this_txn == trx->sp_level || trx->all == NULL) { + trx->sp_level = NULL; + } } else if (tokudb_debug & TOKUDB_DEBUG_TXN) { TOKUDB_TRACE("nothing to commit %d", all); @@ -824,7 +824,7 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { } static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { - TOKUDB_DBUG_ENTER(""); + TOKUDB_DBUG_ENTER("%u", all); DBUG_PRINT("trans", ("aborting transaction %s", all ? "all" : "stmt")); tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN **txn = all ? &trx->all : &trx->stmt; @@ -835,11 +835,11 @@ static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { } tokudb_cleanup_handlers(trx, this_txn); abort_txn_with_progress(this_txn, thd); - if (this_txn == trx->sp_level) { - trx->sp_level = 0; - } - *txn = 0; + *txn = NULL; trx->sub_sp_level = NULL; + if (this_txn == trx->sp_level || trx->all == NULL) { + trx->sp_level = NULL; + } } else { if (tokudb_debug & TOKUDB_DEBUG_TXN) { From 60dd921463bf72545d67b443e10477392381ea5e Mon Sep 17 00:00:00 2001 From: Rik Prohaska Date: Tue, 24 Feb 2015 14:12:36 -0500 Subject: [PATCH 09/10] DB-817 fix savepoint release and duplicate savepoint name bug --- mysql-test/suite/tokudb.bugs/r/db817.result | 33 ++++++++++++++++++ mysql-test/suite/tokudb.bugs/t/db817.test | 38 +++++++++++++++++++++ storage/tokudb/ha_tokudb.cc | 14 +++++--- storage/tokudb/hatoku_hton.cc | 23 +++++++++---- 4 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 mysql-test/suite/tokudb.bugs/r/db817.result create mode 100644 mysql-test/suite/tokudb.bugs/t/db817.test diff --git a/mysql-test/suite/tokudb.bugs/r/db817.result b/mysql-test/suite/tokudb.bugs/r/db817.result new file mode 100644 index 00000000000..d69f0dabcb3 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/r/db817.result @@ -0,0 +1,33 @@ +drop table if exists ti; +create table ti (id int primary key) engine=innodb; +begin; +insert into ti values (0); +savepoint b; +insert into ti values (1); +savepoint a2; +insert into ti values (2); +savepoint b; +insert into ti values (3); +rollback to a2; +commit; +select * from ti; +id +0 +1 +drop table if exists tt; +create table tt (id int primary key) engine=tokudb; +begin; +insert into tt values (0); +savepoint b; +insert into tt values (1); +savepoint a2; +insert into tt values (2); +savepoint b; +insert into tt values (3); +rollback to a2; +commit; +select * from tt; +id +0 +1 +drop table ti,tt; diff --git a/mysql-test/suite/tokudb.bugs/t/db817.test b/mysql-test/suite/tokudb.bugs/t/db817.test new file mode 100644 index 00000000000..53c9edc3893 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/t/db817.test @@ -0,0 +1,38 @@ +# verify that duplicate savepoint names in innodb and tokudb work the same +source include/have_innodb.inc; +source include/have_tokudb.inc; +disable_warnings; +drop table if exists ti; +enable_warnings; +create table ti (id int primary key) engine=innodb; +begin; +insert into ti values (0); +savepoint b; +insert into ti values (1); +savepoint a2; +insert into ti values (2); +savepoint b; +insert into ti values (3); +rollback to a2; +commit; +select * from ti; + +disable_warnings; +drop table if exists tt; +enable_warnings; +create table tt (id int primary key) engine=tokudb; +begin; +insert into tt values (0); +savepoint b; +insert into tt values (1); +savepoint a2; +insert into tt values (2); +savepoint b; +insert into tt values (3); +rollback to a2; +commit; +select * from tt; + +drop table ti,tt; + + diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index c7b3302c248..b348b3bd599 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -3926,13 +3926,13 @@ int ha_tokudb::write_row(uchar * record) { goto cleanup; } } - txn = create_sub_trans ? sub_trans : transaction; - + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_HANDLER_TRACE("txn %p", txn); + } if (tokudb_debug & TOKUDB_DEBUG_CHECK_KEY) { test_row_packing(record,&prim_key,&row); } - if (loader) { error = loader->put(loader, &prim_key, &row); if (error) { @@ -4206,7 +4206,7 @@ int ha_tokudb::delete_row(const uchar * record) { bool has_null; THD* thd = ha_thd(); uint curr_num_DBs; - tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);; + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); ha_statistic_increment(&SSV::ha_delete_count); @@ -4231,10 +4231,14 @@ int ha_tokudb::delete_row(const uchar * record) { goto cleanup; } + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_HANDLER_TRACE("all %p stmt %p sub_sp_level %p transaction %p", trx->all, trx->stmt, trx->sub_sp_level, transaction); + } + error = db_env->del_multiple( db_env, share->key_file[primary_key], - transaction, + transaction, &prim_key, &row, curr_num_DBs, diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index 23b3fd028ed..c3dcd7be221 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -946,7 +946,7 @@ cleanup: #endif static int tokudb_savepoint(handlerton * hton, THD * thd, void *savepoint) { - TOKUDB_DBUG_ENTER(""); + TOKUDB_DBUG_ENTER("%p", savepoint); int error; SP_INFO save_info = (SP_INFO)savepoint; tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); @@ -967,6 +967,9 @@ static int tokudb_savepoint(handlerton * hton, THD * thd, void *savepoint) { trx->sp_level = save_info->txn; save_info->in_sub_stmt = false; } + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_TRACE("begin txn %p", save_info->txn); + } save_info->trx = trx; error = 0; cleanup: @@ -974,7 +977,7 @@ cleanup: } static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *savepoint) { - TOKUDB_DBUG_ENTER(""); + TOKUDB_DBUG_ENTER("%p", savepoint); int error; SP_INFO save_info = (SP_INFO)savepoint; DB_TXN* parent = NULL; @@ -982,6 +985,9 @@ static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *save tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); parent = txn_to_rollback->parent; + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_TRACE("rollback txn %p", txn_to_rollback); + } if (!(error = txn_to_rollback->abort(txn_to_rollback))) { if (save_info->in_sub_stmt) { trx->sub_sp_level = parent; @@ -995,24 +1001,27 @@ static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *save } static int tokudb_release_savepoint(handlerton * hton, THD * thd, void *savepoint) { - TOKUDB_DBUG_ENTER(""); - int error; - + TOKUDB_DBUG_ENTER("%p", savepoint); + int error = 0; SP_INFO save_info = (SP_INFO)savepoint; DB_TXN* parent = NULL; DB_TXN* txn_to_commit = save_info->txn; tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); parent = txn_to_commit->parent; - if (!(error = txn_to_commit->commit(txn_to_commit, 0))) { + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_TRACE("commit txn %p", txn_to_commit); + } + DB_TXN *child = txn_to_commit->get_child(txn_to_commit); + if (child == NULL && !(error = txn_to_commit->commit(txn_to_commit, 0))) { if (save_info->in_sub_stmt) { trx->sub_sp_level = parent; } else { trx->sp_level = parent; } - save_info->txn = NULL; } + save_info->txn = NULL; TOKUDB_DBUG_RETURN(error); } From 135dcf67d875f57a730030fde56fb3844b252da1 Mon Sep 17 00:00:00 2001 From: Rik Prohaska Date: Thu, 26 Feb 2015 13:22:50 -0500 Subject: [PATCH 10/10] DB-805 DB-806 the tokudb loader requires a transaction --- storage/tokudb/ha_tokudb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index b348b3bd599..7c24cc1da2f 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -3236,7 +3236,7 @@ void ha_tokudb::start_bulk_insert(ha_rows rows) { lock_count = 0; if ((rows == 0 || rows > 1) && share->try_table_lock) { - if (get_prelock_empty(thd) && may_table_be_empty(transaction)) { + if (get_prelock_empty(thd) && may_table_be_empty(transaction) && transaction != NULL) { if (using_ignore || is_insert_ignore(thd) || thd->lex->duplicates != DUP_ERROR) { acquire_table_lock(transaction, lock_write); }