diff --git a/mysql-test/suite/innodb_fts/r/crash_recovery.result b/mysql-test/suite/innodb_fts/r/crash_recovery.result index 7d596684344..44d3521df98 100644 --- a/mysql-test/suite/innodb_fts/r/crash_recovery.result +++ b/mysql-test/suite/innodb_fts/r/crash_recovery.result @@ -41,9 +41,11 @@ INSERT INTO articles (title,body) VALUES ('1001 MySQL Tricks','1. Never run mysqld as root. 2. ...'), ('MySQL vs. YourSQL','In the following database comparison ...'), ('MySQL Security','When configured properly, MySQL ...'); +connect dml, localhost, root,,; BEGIN; INSERT INTO articles (title,body) VALUES ('MySQL Tutorial','DBMS stands for DataBase ...'); +connection default; # Make durable the AUTO_INCREMENT in the above incomplete transaction. connect flush_redo_log,localhost,root,,; SET GLOBAL innodb_flush_log_at_trx_commit=1; @@ -52,6 +54,7 @@ DELETE FROM articles LIMIT 1; ROLLBACK; disconnect flush_redo_log; connection default; +disconnect dml; INSERT INTO articles (title,body) VALUES ('MySQL Tutorial','DBMS stands for DataBase ...'); SELECT * FROM articles @@ -79,9 +82,39 @@ INSERT INTO articles VALUES (4, 11, '1001 MySQL Tricks','1. Never run mysqld as root. 2. ...'), (5, 6, 'MySQL vs. YourSQL','In the following database comparison ...'), (7, 4, 'MySQL Security','When configured properly, MySQL ...'); +connect dml, localhost, root,,; BEGIN; INSERT INTO articles VALUES (100, 200, 'MySQL Tutorial','DBMS stands for DataBase ...'); +connect dml2, localhost, root,,; +# +# MDEV-19073 FTS row mismatch after crash recovery +# +CREATE TABLE mdev19073(id SERIAL, title VARCHAR(200), body TEXT, +FULLTEXT(title,body)) ENGINE=InnoDB; +INSERT INTO mdev19073 (title, body) VALUES +('MySQL Tutorial', 'DBMS stands for Database...'); +CREATE FULLTEXT INDEX idx ON mdev19073(title, body); +CREATE TABLE mdev19073_2 LIKE mdev19073; +INSERT INTO mdev19073_2 (title, body) VALUES +('MySQL Tutorial', 'DBMS stands for Database...'); +INSERT INTO mdev19073 (title, body) VALUES +('MariaDB Tutorial', 'DB means Database ...'); +INSERT INTO mdev19073_2 (title, body) VALUES +('MariaDB Tutorial', 'DB means Database ...'); +SELECT * FROM mdev19073 WHERE MATCH (title, body) +AGAINST ('Database' IN NATURAL LANGUAGE MODE); +id title body +1 MySQL Tutorial DBMS stands for Database... +2 MariaDB Tutorial DB means Database ... +SELECT * FROM mdev19073_2 WHERE MATCH (title, body) +AGAINST ('Database' IN NATURAL LANGUAGE MODE); +id title body +1 MySQL Tutorial DBMS stands for Database... +2 MariaDB Tutorial DB means Database ... +connection default; +disconnect dml; +disconnect dml2; INSERT INTO articles VALUES (8, 12, 'MySQL Tutorial','DBMS stands for DataBase ...'); SELECT * FROM articles WHERE MATCH (title, body) AGAINST ('Tutorial' IN NATURAL LANGUAGE MODE); @@ -90,3 +123,14 @@ id FTS_DOC_ID title body 1 10 MySQL Tutorial DBMS stands for DataBase ... 8 12 MySQL Tutorial DBMS stands for DataBase ... DROP TABLE articles; +SELECT * FROM mdev19073 WHERE MATCH (title, body) +AGAINST ('Database' IN NATURAL LANGUAGE MODE); +id title body +1 MySQL Tutorial DBMS stands for Database... +2 MariaDB Tutorial DB means Database ... +SELECT * FROM mdev19073_2 WHERE MATCH (title, body) +AGAINST ('Database' IN NATURAL LANGUAGE MODE); +id title body +1 MySQL Tutorial DBMS stands for Database... +2 MariaDB Tutorial DB means Database ... +DROP TABLE mdev19073, mdev19073_2; diff --git a/mysql-test/suite/innodb_fts/t/crash_recovery.test b/mysql-test/suite/innodb_fts/t/crash_recovery.test index f39d6680dfb..1b321af236a 100644 --- a/mysql-test/suite/innodb_fts/t/crash_recovery.test +++ b/mysql-test/suite/innodb_fts/t/crash_recovery.test @@ -6,6 +6,7 @@ --source include/have_innodb.inc # The embedded server tests do not support restarting. --source include/not_embedded.inc +--source include/maybe_debug.inc FLUSH TABLES; # Following are test for crash recovery on FTS index, the first scenario @@ -73,10 +74,12 @@ INSERT INTO articles (title,body) VALUES ('MySQL vs. YourSQL','In the following database comparison ...'), ('MySQL Security','When configured properly, MySQL ...'); +connect(dml, localhost, root,,); BEGIN; INSERT INTO articles (title,body) VALUES ('MySQL Tutorial','DBMS stands for DataBase ...'); +connection default; --echo # Make durable the AUTO_INCREMENT in the above incomplete transaction. --connect (flush_redo_log,localhost,root,,) @@ -89,6 +92,8 @@ ROLLBACK; --source include/restart_mysqld.inc +disconnect dml; + # This insert will re-initialize the Doc ID counter, it should not crash INSERT INTO articles (title,body) VALUES ('MySQL Tutorial','DBMS stands for DataBase ...'); @@ -121,6 +126,7 @@ INSERT INTO articles VALUES (5, 6, 'MySQL vs. YourSQL','In the following database comparison ...'), (7, 4, 'MySQL Security','When configured properly, MySQL ...'); +connect(dml, localhost, root,,); BEGIN; # Below we do not depend on the durability of the AUTO_INCREMENT sequence, @@ -128,7 +134,49 @@ BEGIN; INSERT INTO articles VALUES (100, 200, 'MySQL Tutorial','DBMS stands for DataBase ...'); +connect(dml2, localhost, root,,); + +--echo # +--echo # MDEV-19073 FTS row mismatch after crash recovery +--echo # + +CREATE TABLE mdev19073(id SERIAL, title VARCHAR(200), body TEXT, + FULLTEXT(title,body)) ENGINE=InnoDB; +INSERT INTO mdev19073 (title, body) VALUES + ('MySQL Tutorial', 'DBMS stands for Database...'); +CREATE FULLTEXT INDEX idx ON mdev19073(title, body); +CREATE TABLE mdev19073_2 LIKE mdev19073; +if ($have_debug) +{ +--disable_query_log +SET @saved_dbug = @@debug_dbug; +SET DEBUG_DBUG = '+d,fts_instrument_sync_debug'; +--enable_query_log +} +INSERT INTO mdev19073_2 (title, body) VALUES + ('MySQL Tutorial', 'DBMS stands for Database...'); +if ($have_debug) +{ +--disable_query_log +SET DEBUG_DBUG = @saved_dbug; +--enable_query_log +} + +INSERT INTO mdev19073 (title, body) VALUES + ('MariaDB Tutorial', 'DB means Database ...'); +INSERT INTO mdev19073_2 (title, body) VALUES + ('MariaDB Tutorial', 'DB means Database ...'); + +# Should return 2 rows +SELECT * FROM mdev19073 WHERE MATCH (title, body) +AGAINST ('Database' IN NATURAL LANGUAGE MODE); +SELECT * FROM mdev19073_2 WHERE MATCH (title, body) +AGAINST ('Database' IN NATURAL LANGUAGE MODE); + +connection default; --source include/restart_mysqld.inc +disconnect dml; +disconnect dml2; # This would re-initialize the FTS index and do the re-tokenization # of above records @@ -138,3 +186,10 @@ SELECT * FROM articles WHERE MATCH (title, body) AGAINST ('Tutorial' IN NATURAL LANGUAGE MODE); DROP TABLE articles; + +# Should return 2 rows +SELECT * FROM mdev19073 WHERE MATCH (title, body) +AGAINST ('Database' IN NATURAL LANGUAGE MODE); +SELECT * FROM mdev19073_2 WHERE MATCH (title, body) +AGAINST ('Database' IN NATURAL LANGUAGE MODE); +DROP TABLE mdev19073, mdev19073_2; diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index fc7436e97c8..06432f091f5 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -43,8 +43,7 @@ Created 4/24/1996 Heikki Tuuri #include "rem0cmp.h" #include "srv0start.h" #include "srv0srv.h" -#include -#include +#include "fts0opt.h" /** Following are the InnoDB system tables. The positions in this array are referenced by enum dict_system_table_id. */ @@ -3121,8 +3120,12 @@ func_exit: FTS */ fts_optimize_remove_table(table); fts_free(table); - } else { + } else if (fts_optimize_wq) { fts_optimize_add_table(table); + } else { + /* fts_optimize_thread is not started yet. + So make the table as non-evictable from cache. */ + dict_table_move_from_lru_to_non_lru(table); } } diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 654f6fc5449..458f4277988 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -2712,6 +2712,10 @@ retry: } if (read_only) { + /* InnoDB stores actual synced_doc_id value + 1 in + FTS_CONFIG table. Reduce the value by 1 while reading + after startup. */ + if (*doc_id) *doc_id -= 1; goto func_exit; } @@ -5388,11 +5392,11 @@ fts_t::fts_t( const dict_table_t* table, mem_heap_t* heap) : - in_queue(0), added_synced(0), dict_locked(0), + added_synced(0), dict_locked(0), bg_threads(0), add_wq(NULL), cache(NULL), - doc_col(ULINT_UNDEFINED), + doc_col(ULINT_UNDEFINED), in_queue(false), fts_heap(heap) { ut_a(table->fts == NULL); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 4b0fbf933e8..202bd80d20a 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -35,9 +35,10 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "srv0start.h" #include "ut0list.h" #include "zlib.h" +#include "fts0opt.h" /** The FTS optimize thread's work queue. */ -static ib_wqueue_t* fts_optimize_wq; +ib_wqueue_t* fts_optimize_wq; /** The FTS vector to store fts_slot_t */ static ib_vector_t* fts_slots; @@ -168,8 +169,8 @@ struct fts_encode_t { /** We use this information to determine when to start the optimize cycle for a table. */ struct fts_slot_t { - /** table identifier, or 0 if the slot is empty */ - table_id_t table_id; + /** table, or NULL if the slot is unused */ + dict_table_t* table; /** whether this slot is being processed */ bool running; @@ -2391,14 +2392,7 @@ fts_optimize_table_bk( return(DB_SUCCESS); } - dict_table_t* table = dict_table_open_on_id( - slot->table_id, FALSE, DICT_TABLE_OP_NORMAL); - - if (!table) { - slot->last_run = now; - return DB_SUCCESS; - } - + dict_table_t* table = slot->table; dberr_t error; if (fil_table_accessible(table) @@ -2418,8 +2412,6 @@ fts_optimize_table_bk( error = DB_SUCCESS; } - dict_table_close(table, FALSE, FALSE); - return(error); } /*********************************************************************//** @@ -2564,11 +2556,13 @@ void fts_optimize_add_table(dict_table_t* table) msg = fts_optimize_create_msg(FTS_MSG_ADD_TABLE, table); - ib_wqueue_add(fts_optimize_wq, msg, msg->heap); + mutex_enter(&fts_optimize_wq->mutex); + + ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true); - mutex_enter(&table->fts->bg_threads_mutex); table->fts->in_queue = true; - mutex_exit(&table->fts->bg_threads_mutex); + + mutex_exit(&fts_optimize_wq->mutex); } /**********************************************************************//** @@ -2595,12 +2589,10 @@ fts_optimize_remove_table( return; } - fts_t* fts = table->fts; - mutex_enter(&fts->bg_threads_mutex); - bool is_in_optimize_queue = fts->in_queue; - mutex_exit(&fts->bg_threads_mutex); + mutex_enter(&fts_optimize_wq->mutex); - if (!is_in_optimize_queue) { + if (!table->fts->in_queue) { + mutex_exit(&fts_optimize_wq->mutex); return; } @@ -2616,15 +2608,17 @@ fts_optimize_remove_table( remove->event = event; msg->ptr = remove; - ib_wqueue_add(fts_optimize_wq, msg, msg->heap); + ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true); + + mutex_exit(&fts_optimize_wq->mutex); os_event_wait(event); os_event_destroy(event); - mutex_enter(&fts->bg_threads_mutex); - fts->in_queue = false; - mutex_exit(&fts->bg_threads_mutex); + ut_d(mutex_enter(&fts_optimize_wq->mutex)); + ut_ad(!table->fts->in_queue); + ut_d(mutex_exit(&fts_optimize_wq->mutex)); } /** Send sync fts cache for the table. @@ -2633,9 +2627,6 @@ void fts_optimize_request_sync_table( dict_table_t* table) { - fts_msg_t* msg; - table_id_t* table_id; - /* if the optimize system not yet initialized, return */ if (!fts_optimize_wq) { return; @@ -2648,39 +2639,36 @@ fts_optimize_request_sync_table( return; } - msg = fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, NULL); + fts_msg_t* msg = fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, table); - table_id = static_cast( - mem_heap_alloc(msg->heap, sizeof(table_id_t))); - *table_id = table->id; - msg->ptr = table_id; + mutex_enter(&fts_optimize_wq->mutex); - ib_wqueue_add(fts_optimize_wq, msg, msg->heap); + ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true); - mutex_enter(&table->fts->bg_threads_mutex); table->fts->in_queue = true; - mutex_exit(&table->fts->bg_threads_mutex); + + mutex_exit(&fts_optimize_wq->mutex); } /** Add a table to fts_slots if it doesn't already exist. */ static bool fts_optimize_new_table(dict_table_t* table) { + ut_ad(table); + ulint i; fts_slot_t* slot; fts_slot_t* empty = NULL; - const table_id_t table_id = table->id; - ut_ad(table_id); /* Search for duplicates, also find a free slot if one exists. */ for (i = 0; i < ib_vector_size(fts_slots); ++i) { slot = static_cast(ib_vector_get(fts_slots, i)); - if (!slot->table_id) { + if (!slot->table) { empty = slot; - } else if (slot->table_id == table_id) { + } else if (slot->table == table) { /* Already exists in our optimize queue. */ - return(FALSE); + return false; } } @@ -2689,36 +2677,35 @@ static bool fts_optimize_new_table(dict_table_t* table) memset(slot, 0x0, sizeof(*slot)); - slot->table_id = table->id; - slot->running = false; - - return(TRUE); + slot->table = table; + return true; } /** Remove a table from fts_slots if it exists. @param[in,out] table table to be removed from fts_slots */ static bool fts_optimize_del_table(const dict_table_t* table) { - const table_id_t table_id = table->id; - ut_ad(table_id); - + ut_ad(table); for (ulint i = 0; i < ib_vector_size(fts_slots); ++i) { fts_slot_t* slot; slot = static_cast(ib_vector_get(fts_slots, i)); - if (slot->table_id == table_id) { + if (slot->table == table) { if (fts_enable_diag_print) { ib::info() << "FTS Optimize Removing table " << table->name; } - slot->table_id = 0; - return(TRUE); + mutex_enter(&fts_optimize_wq->mutex); + slot->table->fts->in_queue = false; + mutex_exit(&fts_optimize_wq->mutex); + slot->table = NULL; + return true; } } - return(FALSE); + return false; } /**********************************************************************//** @@ -2732,7 +2719,7 @@ static ulint fts_optimize_how_many() for (ulint i = 0; i < ib_vector_size(fts_slots); ++i) { const fts_slot_t* slot = static_cast( ib_vector_get_const(fts_slots, i)); - if (slot->table_id == 0) { + if (!slot->table) { continue; } @@ -2768,22 +2755,14 @@ static bool fts_is_sync_needed() const fts_slot_t* slot = static_cast( ib_vector_get_const(fts_slots, i)); - if (slot->table_id == 0) { + if (!slot->table) { continue; } - dict_table_t* table = dict_table_open_on_id( - slot->table_id, FALSE, DICT_TABLE_OP_NORMAL); - if (!table) { - continue; + if (slot->table->fts && slot->table->fts->cache) { + total_memory += slot->table->fts->cache->total_size; } - if (table->fts && table->fts->cache) { - total_memory += table->fts->cache->total_size; - } - - dict_table_close(table, FALSE, FALSE); - if (total_memory > fts_max_total_cache_size) { return(true); } @@ -2793,22 +2772,14 @@ static bool fts_is_sync_needed() } /** Sync fts cache of a table -@param[in] table_id table id */ -static void fts_optimize_sync_table(table_id_t table_id) +@param[in,out] table table to be synced */ +static void fts_optimize_sync_table(dict_table_t* table) { - if (dict_table_t* table = dict_table_open_on_id( - table_id, FALSE, DICT_TABLE_OP_NORMAL)) { - if (fil_table_accessible(table) - && table->fts && table->fts->cache) { - fts_sync_table(table, false); - } - - DBUG_EXECUTE_IF( - "ib_optimize_wq_hang", - os_thread_sleep(6000000);); - - dict_table_close(table, FALSE, FALSE); + if (table->fts && table->fts->cache && fil_table_accessible(table)) { + fts_sync_table(table, false); } + + DBUG_EXECUTE_IF("ib_optimize_wq_hang", os_thread_sleep(6000000);); } /**********************************************************************//** @@ -2847,7 +2818,7 @@ fts_optimize_thread( ib_vector_get(fts_slots, current)); /* Handle the case of empty slots. */ - if (slot->table_id) { + if (slot->table) { slot->running = true; fts_optimize_table_bk(slot); } @@ -2906,7 +2877,7 @@ fts_optimize_thread( os_thread_sleep(300000);); fts_optimize_sync_table( - *static_cast(msg->ptr)); + static_cast(msg->ptr)); break; default: @@ -2925,8 +2896,8 @@ fts_optimize_thread( fts_slot_t* slot = static_cast( ib_vector_get(fts_slots, i)); - if (table_id_t table_id = slot->table_id) { - fts_optimize_sync_table(table_id); + if (slot->table) { + fts_optimize_sync_table(slot->table); } } } @@ -2954,7 +2925,6 @@ fts_optimize_init(void) { mem_heap_t* heap; ib_alloc_t* heap_alloc; - dict_table_t* table; ut_ad(!srv_read_only_mode); @@ -2970,31 +2940,25 @@ fts_optimize_init(void) heap_alloc = ib_heap_allocator_create(heap); fts_slots = ib_vector_create(heap_alloc, sizeof(fts_slot_t), 4); - /* Add fts tables to the fts_slots vector which were skipped during restart */ - std::vector table_vector; - std::vector::iterator it; - + /* Add fts tables to fts_slots which could be skipped + during dict_load_table_one() because fts_optimize_thread + wasn't even started. */ mutex_enter(&dict_sys->mutex); - for (table = UT_LIST_GET_FIRST(dict_sys->table_LRU); - table != NULL; - table = UT_LIST_GET_NEXT(table_LRU, table)) { - if (table->fts && - dict_table_has_fts_index(table)) { - if (fts_optimize_new_table(table)){ - table_vector.push_back(table); - } + for (dict_table_t* table = UT_LIST_GET_FIRST(dict_sys->table_LRU); + table != NULL; + table = UT_LIST_GET_NEXT(table_LRU, table)) { + if (!table->fts || !dict_table_has_fts_index(table)) { + continue; } - } - /* It is better to call dict_table_prevent_eviction() - outside the above loop because it operates on - dict_sys->table_LRU list.*/ - for (it=table_vector.begin();it!=table_vector.end();++it) { - dict_table_prevent_eviction(*it); + /* fts_optimize_thread is not started yet. So there is no + need to acquire fts_optimize_wq->mutex for adding the fts + table to the fts slots. */ + ut_ad(!table->can_be_evicted); + fts_optimize_new_table(table); + table->fts->in_queue = true; } - mutex_exit(&dict_sys->mutex); - table_vector.clear(); fts_opt_shutdown_event = os_event_create(0); last_check_sync_time = time(NULL); diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 813b94cf529..e8a91b0ef55 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -320,9 +320,6 @@ public: /** Mutex protecting bg_threads* and fts_add_wq. */ ib_mutex_t bg_threads_mutex; - /** Whether the table was added to fts_optimize_wq(); - protected by bg_threads_mutex */ - unsigned in_queue:1; /** Whether the ADDED table record sync-ed after crash recovery; protected by bg_threads_mutex */ unsigned added_synced:1; @@ -348,6 +345,10 @@ public: /** Vector of FTS indexes, this is mainly for caching purposes. */ ib_vector_t* indexes; + /** Whether the table exists in fts_optimize_wq; + protected by fts_optimize_wq mutex */ + bool in_queue; + /** Heap for fts_t allocation. */ mem_heap_t* fts_heap; }; diff --git a/storage/innobase/include/fts0opt.h b/storage/innobase/include/fts0opt.h index 4b4d1c4c84a..c527ad8e528 100644 --- a/storage/innobase/include/fts0opt.h +++ b/storage/innobase/include/fts0opt.h @@ -25,6 +25,9 @@ Created 2011-02-15 Jimmy Yang #ifndef INNODB_FTS0OPT_H #define INNODB_FTS0OPT_H +/** The FTS optimize thread's work queue. */ +extern ib_wqueue_t* fts_optimize_wq; + /******************************************************************** Callback function to fetch the rows in an FTS INDEX record. */ ibool diff --git a/storage/innobase/include/ut0wqueue.h b/storage/innobase/include/ut0wqueue.h index 6a096a36894..5a895f4ea3c 100644 --- a/storage/innobase/include/ut0wqueue.h +++ b/storage/innobase/include/ut0wqueue.h @@ -38,7 +38,18 @@ processing. // Forward declaration struct ib_list_t; -struct ib_wqueue_t; + +/** Work queue */ +struct ib_wqueue_t +{ + /** Mutex protecting everything */ + ib_mutex_t mutex; + /** Work item list */ + ib_list_t* items; + /** event we use to signal additions to list; + os_event_set() and os_event_reset() are protected by the mutex */ + os_event_t event; +}; /****************************************************************//** Create a new work queue. @@ -54,15 +65,14 @@ ib_wqueue_free( /*===========*/ ib_wqueue_t* wq); /*!< in: work queue */ -/****************************************************************//** -Add a work item to the queue. */ +/** Add a work item to the queue. +@param[in,out] wq work queue +@param[in] item work item +@param[in,out] heap memory heap to use for allocating list node +@param[in] wq_locked work queue mutex locked */ void -ib_wqueue_add( -/*==========*/ - ib_wqueue_t* wq, /*!< in: work queue */ - void* item, /*!< in: work item */ - mem_heap_t* heap); /*!< in: memory heap to use for - allocating the list node */ +ib_wqueue_add(ib_wqueue_t* wq, void* item, mem_heap_t* heap, + bool wq_locked = false); /** Check if queue is empty. @param wq wait queue @@ -101,5 +111,4 @@ ib_wqueue_len( /*==========*/ ib_wqueue_t* wq); /*mutex); + if (!wq_locked) { + mutex_enter(&wq->mutex); + } ib_list_add_last(wq->items, item, heap); os_event_set(wq->event); - mutex_exit(&wq->mutex); + if (!wq_locked) { + mutex_exit(&wq->mutex); + } } /****************************************************************//** diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index f1848c2b70a..750116ac91e 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -46,6 +46,7 @@ Created 4/24/1996 Heikki Tuuri #include "dict0priv.h" #include "ha_prototypes.h" /* innobase_casedn_str() */ #include "fts0priv.h" +#include "fts0opt.h" /** Following are the InnoDB system tables. The positions in this array are referenced by enum dict_system_table_id. */ @@ -2570,8 +2571,12 @@ func_exit: FTS */ fts_optimize_remove_table(table); fts_free(table); - } else { + } else if (fts_optimize_wq) { fts_optimize_add_table(table); + } else { + /* fts_optimize_thread is not started yet. + So make the table as non-evictable from cache. */ + dict_table_move_from_lru_to_non_lru(table); } } diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc index 1cf5c818a9e..6dbe5e0e2a0 100644 --- a/storage/xtradb/fts/fts0fts.cc +++ b/storage/xtradb/fts/fts0fts.cc @@ -2739,6 +2739,10 @@ retry: } if (read_only) { + /* InnoDB stores actual synced_doc_id value + 1 in + FTS_CONFIG table. Reduce the value by 1 while reading + after startup. */ + if (*doc_id) *doc_id -= 1; goto func_exit; } diff --git a/storage/xtradb/fts/fts0opt.cc b/storage/xtradb/fts/fts0opt.cc index 4472482da81..7f498443544 100644 --- a/storage/xtradb/fts/fts0opt.cc +++ b/storage/xtradb/fts/fts0opt.cc @@ -34,6 +34,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "ut0wqueue.h" #include "srv0start.h" #include "zlib.h" +#include "fts0opt.h" #ifndef UNIV_NONINL #include "fts0types.ic" @@ -41,7 +42,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #endif /** The FTS optimize thread's work queue. */ -static ib_wqueue_t* fts_optimize_wq; +ib_wqueue_t* fts_optimize_wq; /** The FTS vector to store fts_slot_t */ static ib_vector_t* fts_slots; @@ -169,8 +170,8 @@ struct fts_encode_t { /** We use this information to determine when to start the optimize cycle for a table. */ struct fts_slot_t { - /** table identifier, or 0 if the slot is empty */ - table_id_t table_id; + /** table, or NULL if the slot is unused */ + dict_table_t* table; /** whether this slot is being processed */ bool running; @@ -2456,14 +2457,7 @@ fts_optimize_table_bk( return(DB_SUCCESS); } - dict_table_t* table = dict_table_open_on_id( - slot->table_id, FALSE, DICT_TABLE_OP_NORMAL); - - if (!table) { - slot->last_run = now; - return DB_SUCCESS; - } - + dict_table_t* table = slot->table; dberr_t error; if (fil_table_accessible(table) @@ -2483,8 +2477,6 @@ fts_optimize_table_bk( error = DB_SUCCESS; } - dict_table_close(table, FALSE, FALSE); - return(error); } /*********************************************************************//** @@ -2627,11 +2619,13 @@ UNIV_INTERN void fts_optimize_add_table(dict_table_t* table) msg = fts_optimize_create_msg(FTS_MSG_ADD_TABLE, table); - ib_wqueue_add(fts_optimize_wq, msg, msg->heap); + mutex_enter(&fts_optimize_wq->mutex); + + ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true); - mutex_enter(&table->fts->bg_threads_mutex); table->fts->in_queue = true; - mutex_exit(&table->fts->bg_threads_mutex); + + mutex_exit(&fts_optimize_wq->mutex); } /**********************************************************************//** @@ -2648,7 +2642,7 @@ fts_optimize_remove_table( fts_msg_del_t* remove; /* if the optimize system not yet initialized, return */ - if (!fts_optimize_is_init()) { + if (!fts_optimize_wq) { return; } @@ -2660,12 +2654,10 @@ fts_optimize_remove_table( return; } - fts_t* fts = table->fts; - mutex_enter(&fts->bg_threads_mutex); - bool is_in_optimize_queue = fts->in_queue; - mutex_exit(&fts->bg_threads_mutex); + mutex_enter(&fts_optimize_wq->mutex); - if (!is_in_optimize_queue) { + if (!table->fts->in_queue) { + mutex_exit(&fts_optimize_wq->mutex); return; } @@ -2681,15 +2673,17 @@ fts_optimize_remove_table( remove->event = event; msg->ptr = remove; - ib_wqueue_add(fts_optimize_wq, msg, msg->heap); + ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true); + + mutex_exit(&fts_optimize_wq->mutex); os_event_wait(event); os_event_free(event); - mutex_enter(&fts->bg_threads_mutex); - fts->in_queue = false; - mutex_exit(&fts->bg_threads_mutex); + ut_d(mutex_enter(&fts_optimize_wq->mutex)); + ut_ad(!table->fts->in_queue); + ut_d(mutex_exit(&fts_optimize_wq->mutex)); } /** Send sync fts cache for the table. @@ -2700,10 +2694,9 @@ fts_optimize_request_sync_table( dict_table_t* table) { fts_msg_t* msg; - table_id_t* table_id; /* if the optimize system not yet initialized, return */ - if (!fts_optimize_is_init()) { + if (!fts_optimize_wq) { return; } @@ -2715,39 +2708,36 @@ fts_optimize_request_sync_table( return; } - msg = fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, NULL); + msg = fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, table); - table_id = static_cast( - mem_heap_alloc(msg->heap, sizeof(table_id_t))); - *table_id = table->id; - msg->ptr = table_id; + mutex_enter(&fts_optimize_wq->mutex); - ib_wqueue_add(fts_optimize_wq, msg, msg->heap); + ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true); - mutex_enter(&table->fts->bg_threads_mutex); table->fts->in_queue = true; - mutex_exit(&table->fts->bg_threads_mutex); + + mutex_exit(&fts_optimize_wq->mutex); } /** Add a table to fts_slots if it doesn't already exist. */ static bool fts_optimize_new_table(dict_table_t* table) { + ut_ad(table); + ulint i; fts_slot_t* slot; fts_slot_t* empty = NULL; - const table_id_t table_id = table->id; - ut_ad(table_id); /* Search for duplicates, also find a free slot if one exists. */ for (i = 0; i < ib_vector_size(fts_slots); ++i) { slot = static_cast(ib_vector_get(fts_slots, i)); - if (!slot->table_id) { + if (!slot->table) { empty = slot; - } else if (slot->table_id == table_id) { + } else if (slot->table == table) { /* Already exists in our optimize queue. */ - return(FALSE); + return false; } } @@ -2756,37 +2746,36 @@ static bool fts_optimize_new_table(dict_table_t* table) memset(slot, 0x0, sizeof(*slot)); - slot->table_id = table->id; - slot->running = false; - - return(TRUE); + slot->table = table; + return true; } /** Remove a table from fts_slots if it exists. @param[in,out] table table to be removed from fts_slots */ static bool fts_optimize_del_table(const dict_table_t* table) { - const table_id_t table_id = table->id; - ut_ad(table_id); - for (ulint i = 0; i < ib_vector_size(fts_slots); ++i) { fts_slot_t* slot; slot = static_cast(ib_vector_get(fts_slots, i)); - if (slot->table_id == table_id) { + if (slot->table == table) { if (fts_enable_diag_print) { ib_logf(IB_LOG_LEVEL_INFO, "FTS Optimize Removing table %s", table->name); } - slot->table_id = 0; - return(TRUE); + mutex_enter(&fts_optimize_wq->mutex); + slot->table->fts->in_queue = false; + mutex_exit(&fts_optimize_wq->mutex); + + slot->table = NULL; + return true; } } - return(FALSE); + return false; } /**********************************************************************//** @@ -2800,7 +2789,7 @@ static ulint fts_optimize_how_many() for (ulint i = 0; i < ib_vector_size(fts_slots); ++i) { const fts_slot_t* slot = static_cast( ib_vector_get_const(fts_slots, i)); - if (slot->table_id == 0) { + if (!slot->table) { continue; } @@ -2836,22 +2825,14 @@ static bool fts_is_sync_needed() const fts_slot_t* slot = static_cast( ib_vector_get_const(fts_slots, i)); - if (slot->table_id == 0) { + if (!slot->table) { continue; } - dict_table_t* table = dict_table_open_on_id( - slot->table_id, FALSE, DICT_TABLE_OP_NORMAL); - if (!table) { - continue; + if (slot->table->fts && slot->table->fts->cache) { + total_memory += slot->table->fts->cache->total_size; } - if (table->fts && table->fts->cache) { - total_memory += table->fts->cache->total_size; - } - - dict_table_close(table, FALSE, FALSE); - if (total_memory > fts_max_total_cache_size) { return(true); } @@ -2861,22 +2842,16 @@ static bool fts_is_sync_needed() } /** Sync fts cache of a table -@param[in] table_id table id */ -static void fts_optimize_sync_table(table_id_t table_id) +@param[in,out] table table to be synced */ +static void fts_optimize_sync_table(dict_table_t* table) { - if (dict_table_t* table = dict_table_open_on_id( - table_id, FALSE, DICT_TABLE_OP_NORMAL)) { - if (fil_table_accessible(table) - && table->fts && table->fts->cache) { - fts_sync_table(table, true, false, false); - } - - DBUG_EXECUTE_IF( - "ib_optimize_wq_hang", - os_thread_sleep(6000000);); - - dict_table_close(table, FALSE, FALSE); + if (fil_table_accessible(table) + && table->fts && table->fts->cache) { + fts_sync_table(table, true, false, false); } + + DBUG_EXECUTE_IF("ib_optimize_wq_hang", + os_thread_sleep(6000000);); } /**********************************************************************//** @@ -2918,7 +2893,7 @@ fts_optimize_thread( ib_vector_get(fts_slots, current)); /* Handle the case of empty slots. */ - if (slot->table_id) { + if (slot->table) { slot->running = true; fts_optimize_table_bk(slot); } @@ -2978,7 +2953,7 @@ fts_optimize_thread( os_thread_sleep(300000);); fts_optimize_sync_table( - *static_cast(msg->ptr)); + static_cast(msg->ptr)); break; default: @@ -2997,8 +2972,8 @@ fts_optimize_thread( fts_slot_t* slot = static_cast( ib_vector_get(fts_slots, i)); - if (table_id_t table_id = slot->table_id) { - fts_optimize_sync_table(table_id); + if (slot->table) { + fts_optimize_sync_table(slot->table); } } } @@ -3028,24 +3003,35 @@ fts_optimize_init(void) ut_ad(!srv_read_only_mode); /* For now we only support one optimize thread. */ - ut_a(!fts_optimize_is_init()); + ut_a(!fts_optimize_wq); fts_optimize_wq = ib_wqueue_create(); ut_a(fts_optimize_wq != NULL); last_check_sync_time = time(NULL); - os_thread_create(fts_optimize_thread, fts_optimize_wq, NULL); -} + /* Add fts tables to fts slots which could be skipped + during dict_load_table() because fts_optimize_thread + wasn't even started. */ + mutex_enter(&dict_sys->mutex); -/**********************************************************************//** -Check whether the work queue is initialized. -@return TRUE if optimze queue is initialized. */ -UNIV_INTERN -ibool -fts_optimize_is_init(void) -/*======================*/ -{ - return(fts_optimize_wq != NULL); + for (dict_table_t* table = UT_LIST_GET_FIRST(dict_sys->table_LRU); + table != NULL; + table = UT_LIST_GET_NEXT(table_LRU, table)) { + + if (!table->fts || !dict_table_has_fts_index(table)) { + continue; + } + + /* fts_optimize_thread is not started yet. So there is no + need to acqquire fts_optimize_wq->mutex for adding the fts + table to the fts slots. */ + ut_ad(!table->can_be_evicted); + fts_optimize_new_table(table); + table->fts->in_queue = true; + } + + mutex_exit(&dict_sys->mutex); + os_thread_create(fts_optimize_thread, fts_optimize_wq, NULL); } /**********************************************************************//** diff --git a/storage/xtradb/include/fts0fts.h b/storage/xtradb/include/fts0fts.h index 9c70157a0c8..714f811db27 100644 --- a/storage/xtradb/include/fts0fts.h +++ b/storage/xtradb/include/fts0fts.h @@ -285,9 +285,6 @@ struct fts_t { fts_add_wq. */ ib_mutex_t bg_threads_mutex; - /* Whether the table was added to fts_optimize_wq(); - protected by bg_threads mutex */ - unsigned in_queue:1; /* Whether the ADDED table record sync-ed afer crash recovery; protected by bg_threads mutex */ unsigned added_synced:1; @@ -310,6 +307,11 @@ struct fts_t { ib_vector_t* indexes; /*!< Vector of FTS indexes, this is mainly for caching purposes. */ + + /* Whether the table was added to fts_optimize_wq(); + protected by fts_optimize_wq mutex */ + bool in_queue; + mem_heap_t* fts_heap; /*!< heap for fts_t allocation */ }; @@ -631,14 +633,6 @@ void fts_optimize_init(void); /*====================*/ -/**********************************************************************//** -Check whether the work queue is initialized. -@return TRUE if optimze queue is initialized. */ -UNIV_INTERN -ibool -fts_optimize_is_init(void); -/*======================*/ - /****************************************************************//** Drops index ancillary tables for a FTS index @return DB_SUCCESS or error code */ diff --git a/storage/xtradb/include/fts0opt.h b/storage/xtradb/include/fts0opt.h index e8bd4be95b7..29bb14e2f64 100644 --- a/storage/xtradb/include/fts0opt.h +++ b/storage/xtradb/include/fts0opt.h @@ -25,6 +25,9 @@ Created 2011-02-15 Jimmy Yang #ifndef INNODB_FTS0OPT_H #define INNODB_FTS0OPT_H +/** The FTS optimize thread's work queue. */ +extern ib_wqueue_t* fts_optimize_wq; + /******************************************************************** Callback function to fetch the rows in an FTS INDEX record. */ UNIV_INTERN diff --git a/storage/xtradb/include/ut0wqueue.h b/storage/xtradb/include/ut0wqueue.h index 74985618fa3..136bac1cc18 100644 --- a/storage/xtradb/include/ut0wqueue.h +++ b/storage/xtradb/include/ut0wqueue.h @@ -56,16 +56,15 @@ ib_wqueue_free( /*===========*/ ib_wqueue_t* wq); /*!< in: work queue */ -/****************************************************************//** -Add a work item to the queue. */ +/** Add a work item to the queue. +@param[in,out] wq work queue +@param[in] item work item +@param[in,out] heap memory heap to use for allocating list node +@param[in] wq_locked work queue mutex locked */ UNIV_INTERN void -ib_wqueue_add( -/*==========*/ - ib_wqueue_t* wq, /*!< in: work queue */ - void* item, /*!< in: work item */ - mem_heap_t* heap); /*!< in: memory heap to use for allocating the - list node */ +ib_wqueue_add(ib_wqueue_t* wq, void* item, mem_heap_t* heap, + bool wq_locked = false); /******************************************************************** Check if queue is empty. */ @@ -104,7 +103,6 @@ ib_wqueue_nowait( /*=============*/ ib_wqueue_t* wq); /*mutex); + if (!wq_locked) { + mutex_enter(&wq->mutex); + } ib_list_add_last(wq->items, item, heap); os_event_set(wq->event); - mutex_exit(&wq->mutex); + if (!wq_locked) { + mutex_exit(&wq->mutex); + } } /****************************************************************//**