From 6a04c2a1aa60e67c07d4e6ada63f65cc3b26a5a9 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Tue, 22 May 2018 12:09:05 -0700 Subject: [PATCH 01/15] MDEV-16235 Server crashes in my_utf8_uni or in my_strtod_int upon SELECT .. LIMIT 0 The code must differentiate between a SELECT with contradictory WHERE/HAVING and one with LIMIT 0. Also for the latter printed 'Zero limit' instead of 'Impossible where' in the EXPLAIN output. --- mysql-test/r/limit.result | 16 ++++++++++++++++ mysql-test/r/subselect4.result | 2 +- mysql-test/r/subselect_mat_cost_bugs.result | 2 +- mysql-test/t/limit.test | 14 ++++++++++++++ mysql-test/t/subselect4.test | 2 +- sql/sql_select.cc | 17 +++++++++++++---- 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/mysql-test/r/limit.result b/mysql-test/r/limit.result index 176a93c7a46..deb4ca2ab95 100644 --- a/mysql-test/r/limit.result +++ b/mysql-test/r/limit.result @@ -146,3 +146,19 @@ a 16 DROP TABLE t1; End of 5.1 tests +# +# mdev-16235: SELECT over a table with LIMIT 0 +# +EXPLAIN +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Zero limit +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +start_time user_host query_time lock_time rows_sent rows_examined db last_insert_id insert_id server_id sql_text +EXPLAIN +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Zero limit +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +help_topic_id name help_category_id description example url +End of 5.5 tests diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 6accc23d18a..8973330c7da 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2512,7 +2512,7 @@ SELECT 2 IN (SELECT 2 from DUAL WHERE 1 != 1); SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; # -# mfrv-14515: Wrong results for tableless query with subquery in WHERE +# mdev-14515: Wrong results for tableless query with subquery in WHERE # and implicit aggregation # create table t1 (i1 int, i2 int); diff --git a/mysql-test/r/subselect_mat_cost_bugs.result b/mysql-test/r/subselect_mat_cost_bugs.result index df6b543bab8..d33f1488e4d 100644 --- a/mysql-test/r/subselect_mat_cost_bugs.result +++ b/mysql-test/r/subselect_mat_cost_bugs.result @@ -334,7 +334,7 @@ SELECT * FROM t1 WHERE (f1) IN (SELECT f1 FROM t2) LIMIT 0; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Zero limit 2 DEPENDENT SUBQUERY t2 ALL NULL NULL NULL NULL 2 Using where SELECT * FROM t1 WHERE (f1) IN (SELECT f1 FROM t2) diff --git a/mysql-test/t/limit.test b/mysql-test/t/limit.test index 4dbe13096d4..668d3b74518 100644 --- a/mysql-test/t/limit.test +++ b/mysql-test/t/limit.test @@ -115,3 +115,17 @@ SELECT a FROM t1 ORDER BY a LIMIT 2 OFFSET 14; DROP TABLE t1; --echo End of 5.1 tests + +--echo # +--echo # mdev-16235: SELECT over a table with LIMIT 0 +--echo # + +EXPLAIN +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; + +EXPLAIN +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; + +--echo End of 5.5 tests diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 2b53b55b735..2727fd9daed 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -2047,7 +2047,7 @@ SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; --echo # ---echo # mfrv-14515: Wrong results for tableless query with subquery in WHERE +--echo # mdev-14515: Wrong results for tableless query with subquery in WHERE --echo # and implicit aggregation --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1e9f1c0848b..d6d269a700f 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1136,10 +1136,19 @@ JOIN::optimize() if (cond_value == Item::COND_FALSE || having_value == Item::COND_FALSE || (!unit->select_limit_cnt && !(select_options & OPTION_FOUND_ROWS))) { /* Impossible cond */ - DBUG_PRINT("info", (having_value == Item::COND_FALSE ? - "Impossible HAVING" : "Impossible WHERE")); - zero_result_cause= having_value == Item::COND_FALSE ? - "Impossible HAVING" : "Impossible WHERE"; + if (unit->select_limit_cnt) + { + DBUG_PRINT("info", (having_value == Item::COND_FALSE ? + "Impossible HAVING" : "Impossible WHERE")); + zero_result_cause= having_value == Item::COND_FALSE ? + "Impossible HAVING" : "Impossible WHERE"; + } + else + { + DBUG_PRINT("info", ("Zero limit")); + zero_result_cause= "Zero limit"; + conds= 0; + } table_count= top_join_tab_count= 0; error= 0; goto setup_subq_exit; From 1ada4afb0a51f7283b6187a95019ec2cb80c8a0b Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Sun, 29 Apr 2018 19:47:48 +0300 Subject: [PATCH 02/15] mtr: use process launch -- args to start mysqld in lldb gdb's "set args" equivalent for lldb would be "settings set target.run-args", however it doesn't play well with double dashed args (--). --- mysql-test/mysql-test-run.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index de1871cbf03..ba49d88a85f 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -6051,7 +6051,7 @@ sub lldb_arguments { $input = $input ? "< $input" : ""; # write init file for mysqld or client - mtr_tofile($lldb_init_file, "set args $str $input\n"); + mtr_tofile($lldb_init_file, "process launch --stop-at-entry -- $str $input\n"); print "\nTo start lldb for $type, type in another window:\n"; print "cd $glob_mysql_test_dir && lldb -s $lldb_init_file $$exe\n"; From 5fb2c586f225361deae0f04dc6f72a3f0d168ac2 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Jun 2018 11:52:48 +0530 Subject: [PATCH 03/15] MDEV-16225: wrong resultset from query with semijoin=on For non-semi-join subquery optimization we do a cost based decision between Materialisation and IN -> EXIST transformation. The issue in this case is that for IN->EXIST transformation we run JOIN::reoptimize with the IN->EXISt conditions and we come up with a new query plan. But when we compare the cost with Materialization, we make the decision to chose Materialization so we need to restore the query plan for Materilization. The saving and restoring for keyuse array and join_tab keyuse is only done when we have atleast one element in the keyuse_array , we are now changing to do it even for 0 elements to main the generality. --- mysql-test/r/subselect_sj2_mat.result | 57 +++++++++++++++++++++++++++ mysql-test/t/subselect_sj2_mat.test | 42 ++++++++++++++++++++ sql/sql_select.cc | 46 ++++++++++----------- 3 files changed, 119 insertions(+), 26 deletions(-) diff --git a/mysql-test/r/subselect_sj2_mat.result b/mysql-test/r/subselect_sj2_mat.result index 835742a3ff4..19aa3bd3f02 100644 --- a/mysql-test/r/subselect_sj2_mat.result +++ b/mysql-test/r/subselect_sj2_mat.result @@ -1601,3 +1601,60 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY eq_ref distinct_key distinct_key 11 func 1 2 MATERIALIZED t2 ALL NULL NULL NULL NULL 3 Using where DROP TABLE t1,t2; +# +# MDEV-16225: wrong resultset from query with semijoin=on +# +CREATE TABLE t1 ( +`id` int(10) NOT NULL AUTO_INCREMENT, +`local_name` varchar(64) NOT NULL, +PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=latin1; +insert into t1(`id`,`local_name`) values +(1,'Cash Advance'), +(2,'Cash Advance'), +(3,'Rollover'), +(4,'AL Installment'), +(5,'AL Installment'), +(6,'AL Installment'), +(7,'AL Installment'), +(8,'AL Installment'), +(9,'AL Installment'), +(10,'Internet Payday'), +(11,'Rollover - Internet Payday'), +(12,'AL Monthly Installment'), +(13,'AL Semi-Monthly Installment'); +explain +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( +t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) +OR +(t.id IN (0,4,12,13,1,10,3,11)) +); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t index PRIMARY PRIMARY 4 NULL 13 Using where; Using index +2 MATERIALIZED ALL distinct_key NULL NULL NULL 8 +2 MATERIALIZED A ALL PRIMARY NULL NULL NULL 13 Using where; Using join buffer (flat, BNL join) +3 MATERIALIZED B ALL PRIMARY NULL NULL NULL 13 Using where +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( +t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) +OR +(t.id IN (0,4,12,13,1,10,3,11)) +); +id +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +drop table t1; diff --git a/mysql-test/t/subselect_sj2_mat.test b/mysql-test/t/subselect_sj2_mat.test index cfb6c8c2819..0665cdf68fe 100644 --- a/mysql-test/t/subselect_sj2_mat.test +++ b/mysql-test/t/subselect_sj2_mat.test @@ -303,3 +303,45 @@ eval $q; eval explain $q; DROP TABLE t1,t2; + +--echo # +--echo # MDEV-16225: wrong resultset from query with semijoin=on +--echo # + +CREATE TABLE t1 ( + `id` int(10) NOT NULL AUTO_INCREMENT, + `local_name` varchar(64) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=latin1; + +insert into t1(`id`,`local_name`) values +(1,'Cash Advance'), +(2,'Cash Advance'), +(3,'Rollover'), +(4,'AL Installment'), +(5,'AL Installment'), +(6,'AL Installment'), +(7,'AL Installment'), +(8,'AL Installment'), +(9,'AL Installment'), +(10,'Internet Payday'), +(11,'Rollover - Internet Payday'), +(12,'AL Monthly Installment'), +(13,'AL Semi-Monthly Installment'); + +explain +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( + t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) + OR + (t.id IN (0,4,12,13,1,10,3,11)) +); +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( + t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) + OR + (t.id IN (0,4,12,13,1,10,3,11)) +); +drop table t1; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index d6d269a700f..016dd594e08 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -23739,21 +23739,18 @@ void JOIN::set_allowed_join_cache_types() void JOIN::save_query_plan(Join_plan_state *save_to) { - if (keyuse.elements) - { - DYNAMIC_ARRAY tmp_keyuse; - /* Swap the current and the backup keyuse internal arrays. */ - tmp_keyuse= keyuse; - keyuse= save_to->keyuse; /* keyuse is reset to an empty array. */ - save_to->keyuse= tmp_keyuse; + DYNAMIC_ARRAY tmp_keyuse; + /* Swap the current and the backup keyuse internal arrays. */ + tmp_keyuse= keyuse; + keyuse= save_to->keyuse; /* keyuse is reset to an empty array. */ + save_to->keyuse= tmp_keyuse; - for (uint i= 0; i < table_count; i++) - { - save_to->join_tab_keyuse[i]= join_tab[i].keyuse; - join_tab[i].keyuse= NULL; - save_to->join_tab_checked_keys[i]= join_tab[i].checked_keys; - join_tab[i].checked_keys.clear_all(); - } + for (uint i= 0; i < table_count; i++) + { + save_to->join_tab_keyuse[i]= join_tab[i].keyuse; + join_tab[i].keyuse= NULL; + save_to->join_tab_checked_keys[i]= join_tab[i].checked_keys; + join_tab[i].checked_keys.clear_all(); } memcpy((uchar*) save_to->best_positions, (uchar*) best_positions, sizeof(POSITION) * (table_count + 1)); @@ -23791,20 +23788,17 @@ void JOIN::reset_query_plan() void JOIN::restore_query_plan(Join_plan_state *restore_from) { - if (restore_from->keyuse.elements) + DYNAMIC_ARRAY tmp_keyuse; + tmp_keyuse= keyuse; + keyuse= restore_from->keyuse; + restore_from->keyuse= tmp_keyuse; + + for (uint i= 0; i < table_count; i++) { - DYNAMIC_ARRAY tmp_keyuse; - tmp_keyuse= keyuse; - keyuse= restore_from->keyuse; - restore_from->keyuse= tmp_keyuse; - - for (uint i= 0; i < table_count; i++) - { - join_tab[i].keyuse= restore_from->join_tab_keyuse[i]; - join_tab[i].checked_keys= restore_from->join_tab_checked_keys[i]; - } - + join_tab[i].keyuse= restore_from->join_tab_keyuse[i]; + join_tab[i].checked_keys= restore_from->join_tab_checked_keys[i]; } + memcpy((uchar*) best_positions, (uchar*) restore_from->best_positions, sizeof(POSITION) * (table_count + 1)); /* Restore SJM nests */ From 72b6d01848e56a75349d663bc61bbe71f97a280b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 5 Jun 2018 22:13:19 +0100 Subject: [PATCH 04/15] MDEV-10246 ssl-* config file options have no effect without mysql_ssl_set() Partially revert 4ef74979969ac9339d0d42c11a6f26632e6776f1 that caused regression. Any ssl- option must imply use_ssl=1, even if mysql_set_ssl() was not used. --- sql-common/client.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql-common/client.c b/sql-common/client.c index 6321e64bc75..ec992f80e8d 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -2541,6 +2541,10 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, mysql->client_flag|= CLIENT_MULTI_RESULTS; #ifdef HAVE_OPENSSL + if (mysql->options.ssl_key || mysql->options.ssl_cert || + mysql->options.ssl_ca || mysql->options.ssl_capath || + mysql->options.ssl_cipher) + mysql->options.use_ssl = 1; if (mysql->options.use_ssl) mysql->client_flag|= CLIENT_SSL; #endif /* HAVE_OPENSSL */ From 75b4eb5cc969a1f5fc221d03c62987b8e57d6332 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 6 Jun 2018 15:27:57 +0200 Subject: [PATCH 05/15] Catch of OOM situation. --- sql/sql_base.cc | 3 +++ sql/table.cc | 2 ++ 2 files changed, 5 insertions(+) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index f5864256440..71bbb538b5c 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -6426,6 +6426,9 @@ find_field_in_natural_join(THD *thd, TABLE_LIST *table_ref, const char *name, column reference. See create_view_field() for details. */ item= nj_col->create_item(thd); + if (!item) + DBUG_RETURN(NULL); + /* *ref != NULL means that *ref contains the item that we need to replace. If the item was aliased by the user, set the alias to diff --git a/sql/table.cc b/sql/table.cc index bb4eae9b1e2..552f514283d 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -5252,6 +5252,8 @@ Item *create_view_field(THD *thd, TABLE_LIST *view, Item **field_ref, Item *item= new Item_direct_view_ref(&view->view->select_lex.context, field_ref, view->alias, name, view); + if (!item) + return NULL; /* Force creation of nullable item for the result tmp table for outer joined views/derived tables. From d9b159a2027c56c5c87385cfe1ae43b8c73a97b6 Mon Sep 17 00:00:00 2001 From: Chris Calender Date: Tue, 17 Apr 2018 15:00:15 -0400 Subject: [PATCH 06/15] MDEV-15789 - mysqlslap use incorrect table def The bug arises when one uses --auto-generate-sql-guid-primary (and --auto-generate-sql-secondary-indexes) with mysqlslap and also have sql_mode=STRICT_TRANS_TABLE. When using this option, mysqlslap should create a column with varchar(36), but it appears to create it as a varchar(32) only. Then if one has sql_mode=STRICT_TRANS_TABLES, it throws an error, like: mysqlslap: Cannot run query INSERT INTO t1 VALUES (...) ERROR : Data too long for column 'id' at row 1 Upstream bug report: BUG#80329. --- client/mysqlslap.c | 4 ++-- mysql-test/r/mysqlslap.result | 3 +++ mysql-test/t/mysqlslap.test | 8 ++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index a9746916411..ba7b8cea6a9 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -840,7 +840,7 @@ build_table_string(void) if (auto_generate_sql_guid_primary) { - dynstr_append(&table_string, "id varchar(32) primary key"); + dynstr_append(&table_string, "id varchar(36) primary key"); if (num_int_cols || num_char_cols || auto_generate_sql_guid_primary) dynstr_append(&table_string, ","); @@ -855,7 +855,7 @@ build_table_string(void) if (count) /* Except for the first pass we add a comma */ dynstr_append(&table_string, ","); - if (snprintf(buf, HUGE_STRING_LENGTH, "id%d varchar(32) unique key", count) + if (snprintf(buf, HUGE_STRING_LENGTH, "id%d varchar(36) unique key", count) > HUGE_STRING_LENGTH) { fprintf(stderr, "Memory Allocation error in create table\n"); diff --git a/mysql-test/r/mysqlslap.result b/mysql-test/r/mysqlslap.result index af78677647f..50bd53f4263 100644 --- a/mysql-test/r/mysqlslap.result +++ b/mysql-test/r/mysqlslap.result @@ -255,3 +255,6 @@ Benchmark # MDEV-4684 - Enhancement request: --init-command support for mysqlslap # DROP TABLE t1; +# +# Bug MDEV-15789 (Upstream: #80329): MYSQLSLAP OPTIONS --AUTO-GENERATE-SQL-GUID-PRIMARY and --AUTO-GENERATE-SQL-SECONDARY-INDEXES DONT WORK +# diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index c49c4ab3d7d..81115d59d09 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -80,3 +80,11 @@ DROP DATABASE bug58090; --exec $MYSQL_SLAP --create-schema=test --init-command="CREATE TABLE t1(a INT)" --silent --concurrency=1 --iterations=1 DROP TABLE t1; + +--echo # +--echo # Bug MDEV-15789 (Upstream: #80329): MYSQLSLAP OPTIONS --AUTO-GENERATE-SQL-GUID-PRIMARY and --AUTO-GENERATE-SQL-SECONDARY-INDEXES DONT WORK +--echo # + +--exec $MYSQL_SLAP --concurrency=1 --silent --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --create-schema=slap + +--exec $MYSQL_SLAP --concurrency=1 --silent --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-secondary-indexes=1 --create-schema=slap From b8e267c0c5007dbfebc2dee8a4f408f28b637ba5 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Wed, 9 May 2018 15:14:57 +0300 Subject: [PATCH 07/15] MDEV-15778: Manually backport TokuDB macOS fixes from 10.0 Fix build on macOS 10.13: 39dceaae607 MDEV-10983: TokuDB does not compile on OS X 10.12 Make use of a different function to get the current tid. Additionally, librt doesn't exist on OS X. Use System library instead. storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake | 4 +++- storage/tokudb/PerconaFT/portability/portability.cc | 9 ++++++++- storage/tokudb/PerconaFT/portability/tests/test-xid.cc | 9 ++++++++- storage/tokudb/PerconaFT/portability/toku_config.h.in | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) --- .../ft-index/cmake_modules/TokuFeatureDetection.cmake | 4 +++- storage/tokudb/ft-index/portability/portability.cc | 9 ++++++++- storage/tokudb/ft-index/portability/tests/test-xid.cc | 9 ++++++++- storage/tokudb/ft-index/portability/toku_config.h.in | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake b/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake index e7fd27525d5..ef7934a5a09 100644 --- a/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake +++ b/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake @@ -96,7 +96,7 @@ if (NOT HAVE_BACKTRACE_WITHOUT_EXECINFO) endif () endif () -if(HAVE_CLOCK_REALTIME) +if(HAVE_CLOCK_REALTIME AND (NOT APPLE)) list(APPEND EXTRA_SYSTEM_LIBS rt) else() list(APPEND EXTRA_SYSTEM_LIBS System) @@ -108,6 +108,8 @@ check_function_exists(pthread_rwlockattr_setkind_np HAVE_PTHREAD_RWLOCKATTR_SETK ## check for the right way to yield using pthreads check_function_exists(pthread_yield HAVE_PTHREAD_YIELD) check_function_exists(pthread_yield_np HAVE_PTHREAD_YIELD_NP) +## check if we have pthread_threadid_np() (i.e. osx) +check_function_exists(pthread_threadid_np HAVE_PTHREAD_THREADID_NP) ## check if we have pthread_getthreadid_np() (i.e. freebsd) check_function_exists(pthread_getthreadid_np HAVE_PTHREAD_GETTHREADID_NP) diff --git a/storage/tokudb/ft-index/portability/portability.cc b/storage/tokudb/ft-index/portability/portability.cc index 09c1ccd50be..9863c45c60d 100644 --- a/storage/tokudb/ft-index/portability/portability.cc +++ b/storage/tokudb/ft-index/portability/portability.cc @@ -115,6 +115,9 @@ PATENT RIGHTS GRANT: #if defined(HAVE_SYS_SYSCTL_H) # include #endif +#if defined(HAVE_PTHREAD_H) +# include +#endif #if defined(HAVE_PTHREAD_NP_H) # include #endif @@ -154,7 +157,11 @@ toku_os_getpid(void) { int toku_os_gettid(void) { -#if defined(__NR_gettid) +#if defined(HAVE_PTHREAD_THREADID_NP) + uint64_t result; + pthread_threadid_np(NULL, &result); + return (int) result; // Used for instrumentation so overflow is ok here. +#elif defined(__NR_gettid) return syscall(__NR_gettid); #elif defined(SYS_gettid) return syscall(SYS_gettid); diff --git a/storage/tokudb/ft-index/portability/tests/test-xid.cc b/storage/tokudb/ft-index/portability/tests/test-xid.cc index 9277f984b43..fb1abd33793 100644 --- a/storage/tokudb/ft-index/portability/tests/test-xid.cc +++ b/storage/tokudb/ft-index/portability/tests/test-xid.cc @@ -103,11 +103,18 @@ PATENT RIGHTS GRANT: #if defined(HAVE_PTHREAD_NP_H) # include #endif +#if defined(HAVE_PTHREAD_H) +# include +#endif // since we implement the same thing here as in toku_os_gettid, this test // is pretty pointless static int gettid(void) { -#if defined(__NR_gettid) +#if defined(HAVE_PTHREAD_THREADID_NP) + uint64_t result; + pthread_threadid_np(NULL, &result); + return (int) result; +#elif defined(__NR_gettid) return syscall(__NR_gettid); #elif defined(SYS_gettid) return syscall(SYS_gettid); diff --git a/storage/tokudb/ft-index/portability/toku_config.h.in b/storage/tokudb/ft-index/portability/toku_config.h.in index c56674ece9e..b8a89e81da6 100644 --- a/storage/tokudb/ft-index/portability/toku_config.h.in +++ b/storage/tokudb/ft-index/portability/toku_config.h.in @@ -57,6 +57,7 @@ #cmakedefine HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP 1 #cmakedefine HAVE_PTHREAD_YIELD 1 #cmakedefine HAVE_PTHREAD_YIELD_NP 1 +#cmakedefine HAVE_PTHREAD_THREADID_NP 1 #cmakedefine HAVE_PTHREAD_GETTHREADID_NP 1 #cmakedefine PTHREAD_YIELD_RETURNS_INT 1 From 1735fa340a9d7ca8683f18fe2ecc148423e78ba7 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Wed, 9 May 2018 16:54:16 +0300 Subject: [PATCH 08/15] MDEV-15778: Remove packed attr from omt_ and subtree_ classes This is happening because they are declared as packed and clang has -Waddress-of-packed-member when passing the address of a packed member, a legit concern on different architectures. The easiest way to get rid of the errors is to remove the packed attribute from said structs. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 02f3f0d759a..6cf3f80d2bc 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} __attribute__((__packed__,aligned(4))); +} ; template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} __attribute__((__packed__)) ; +} ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} __attribute__((__packed__,aligned(4))); +} ; template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} __attribute__((__packed__,aligned(4))); +} ; } From 814a284f22b617e2d1ced74ba10d1a870ee6267b Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Thu, 12 Apr 2018 13:33:39 +0300 Subject: [PATCH 09/15] Ignore .cbp QtCreator && CodeBlocks project files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4e3d2a1e66e..0e00a27cbf0 100644 --- a/.gitignore +++ b/.gitignore @@ -236,3 +236,6 @@ storage/mroonga/vendor/groonga/src/grnslap storage/mroonga/vendor/groonga/src/groonga storage/mroonga/vendor/groonga/src/groonga-benchmark storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset + +# QtCreator && CodeBlocks +*.cbp From 0e6d6354bf32ac26d36a5a1bb854f2b22de3a494 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Tue, 15 May 2018 10:25:47 +0300 Subject: [PATCH 10/15] Also ignore macOS .DS_Store Finder junk. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0e00a27cbf0..5f2bf5a0fcd 100644 --- a/.gitignore +++ b/.gitignore @@ -237,5 +237,8 @@ storage/mroonga/vendor/groonga/src/groonga storage/mroonga/vendor/groonga/src/groonga-benchmark storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset +# macOS garbage +.DS_Store + # QtCreator && CodeBlocks *.cbp From d39629f01ebdd5b89186e6c8a4a8d3dd528bd26a Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Mon, 7 May 2018 22:40:27 +0300 Subject: [PATCH 11/15] MDEV-16075: Workaround to run MTR test suite for make test Assign all tests added via MY_ADD_TEST to a bogus default_ignore target, so that they are not ran by default when doing bare make test. Add default test named MTR that calls mysql-test-run suite, which is now the single test run by make test. In consequence, modified unit/suite.pm to exclude the MTR test and run the real ctests flagged for default_ignore target, thus no circular loop. --- CMakeLists.txt | 4 ++++ cmake/ctest.cmake | 2 +- mysql-test/suite/unit/suite.pm | 2 +- unittest/sql/CMakeLists.txt | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ab17ebf64c..21ed1fcc24b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,6 +366,10 @@ INCLUDE(maintainer) IF(WITH_UNIT_TESTS) ENABLE_TESTING() + # This is the only instance where ADD_TEST should be used, + # to make sure that make test will run MTR, + # use MY_ADD_TEST macro to add other tests + ADD_TEST(NAME MTR COMMAND ./mysql-test-run WORKING_DIRECTORY "mysql-test") ADD_SUBDIRECTORY(unittest/mytap) ADD_SUBDIRECTORY(unittest/strings) ADD_SUBDIRECTORY(unittest/examples) diff --git a/cmake/ctest.cmake b/cmake/ctest.cmake index 08852a366f6..fde7e1632f6 100644 --- a/cmake/ctest.cmake +++ b/cmake/ctest.cmake @@ -2,7 +2,7 @@ INCLUDE(${MYSQL_CMAKE_SCRIPT_DIR}/cmake_parse_arguments.cmake) MACRO(MY_ADD_TEST name) - ADD_TEST(${name} ${name}-t) + ADD_TEST(NAME ${name} COMMAND ${name}-t CONFIGURATIONS default_ignore) ENDMACRO() MACRO (MY_ADD_TESTS) diff --git a/mysql-test/suite/unit/suite.pm b/mysql-test/suite/unit/suite.pm index 78d82ccb31d..43eaa970350 100644 --- a/mysql-test/suite/unit/suite.pm +++ b/mysql-test/suite/unit/suite.pm @@ -31,7 +31,7 @@ sub start_test { return "Not run for embedded server" if $::opt_embedded_server; return "Not configured to run ctest" unless -f "../CTestTestfile.cmake"; my ($ctest_vs)= $opt_vs_config ? "--build-config $opt_vs_config" : ""; - my (@ctest_list)= `cd .. && ctest $opt_vs_config --show-only --verbose`; + my (@ctest_list)= `cd .. && ctest $opt_vs_config -E MTR -C default_ignore --show-only --verbose`; return "No ctest" if $?; my ($command, %tests); diff --git a/unittest/sql/CMakeLists.txt b/unittest/sql/CMakeLists.txt index f80f2a5ae70..1efb0ac9cd1 100644 --- a/unittest/sql/CMakeLists.txt +++ b/unittest/sql/CMakeLists.txt @@ -26,4 +26,4 @@ ELSE() ENDIF() TARGET_LINK_LIBRARIES(explain_filename-t sql mytap) -ADD_TEST(explain_filename explain_filename-t) +MY_ADD_TEST(explain_filename explain_filename-t) From 7fca4b50ffbe750532cfcdb95bcd425ec1b8e22b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 15:20:43 +0300 Subject: [PATCH 12/15] Revert "MDEV-15778: Remove packed attr from omt_ and subtree_ classes" This reverts commit 1735fa340a9d7ca8683f18fe2ecc148423e78ba7. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 6cf3f80d2bc..02f3f0d759a 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} ; +} __attribute__((__packed__,aligned(4))); template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} ; +} __attribute__((__packed__)) ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} ; +} __attribute__((__packed__,aligned(4))); template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} ; +} __attribute__((__packed__,aligned(4))); } From 21246066b296bd9c6e44272a8f93cb7dc73ea43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 15:54:57 +0300 Subject: [PATCH 13/15] Make TokuDB compile with GCC-8 GCC-8 introduced multiple warnings and increased the level of strictness. * -Wshadow will warn if a local variable shadows a typedef. * GCC will also warn when memsetting a non-trivial type. In this case a non-trivial type can not have a custom constructor. For all intents and purposes, the class is trivially-copyable. * GCC will also warn if you use too many paranthesses which are not necessary --- storage/tokudb/CMakeLists.txt | 6 ++++++ storage/tokudb/ft-index/portability/memory.h | 2 +- storage/tokudb/ft-index/tools/ba_replay.cc | 2 +- storage/tokudb/ft-index/util/dmt.cc | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 903471b097c..9bff7d4729f 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -17,6 +17,12 @@ IF(NOT LIBJEMALLOC) MESSAGE(WARNING "TokuDB is enabled, but jemalloc is not. This configuration is not supported") ENDIF() +CHECK_C_COMPILER_FLAG("-Wshadow" HAVE_WSHADOW) +IF (HAVE_WSHADOW) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-shadow") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-shadow") +ENDIF() + IF (HAVE_WVLA) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-vla") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-vla") diff --git a/storage/tokudb/ft-index/portability/memory.h b/storage/tokudb/ft-index/portability/memory.h index 837b0a70265..52e0c69575f 100644 --- a/storage/tokudb/ft-index/portability/memory.h +++ b/storage/tokudb/ft-index/portability/memory.h @@ -160,7 +160,7 @@ size_t toku_malloc_usable_size(void *p) __attribute__((__visibility__("default") #define XMALLOC(v) CAST_FROM_VOIDP(v, toku_xmalloc(sizeof(*v))) #define XMALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xmalloc((n)*sizeof(*v))) #define XCALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xcalloc((n), (sizeof(*v)))) -#define XCALLOC(v) XCALLOC_N(1,(v)) +#define XCALLOC(v) XCALLOC_N(1,v) #define XREALLOC(v,s) CAST_FROM_VOIDP(v, toku_xrealloc(v, s)) #define XREALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xrealloc(v, (n)*sizeof(*v))) diff --git a/storage/tokudb/ft-index/tools/ba_replay.cc b/storage/tokudb/ft-index/tools/ba_replay.cc index e274ac0a1e8..8f2bc3a02ec 100644 --- a/storage/tokudb/ft-index/tools/ba_replay.cc +++ b/storage/tokudb/ft-index/tools/ba_replay.cc @@ -638,7 +638,7 @@ int main(int argc, char *argv[]) { it == candidate_strategies.begin() ? &stats : &dummy_stats); struct fragmentation_report aggregate_report; - memset(&aggregate_report, 0, sizeof(aggregate_report)); + memset(static_cast(&aggregate_report), 0, sizeof(aggregate_report)); for (map::iterator rp = reports.begin(); rp != reports.end(); rp++) { const struct fragmentation_report &report = rp->second; diff --git a/storage/tokudb/ft-index/util/dmt.cc b/storage/tokudb/ft-index/util/dmt.cc index 3e0b512d7a7..1a7e2db9f40 100644 --- a/storage/tokudb/ft-index/util/dmt.cc +++ b/storage/tokudb/ft-index/util/dmt.cc @@ -132,8 +132,8 @@ void dmt::create_from_sorted_memory_of_fix paranoid_invariant(numvalues > 0); void *ptr = toku_mempool_malloc(&this->mp, aligned_memsize); paranoid_invariant_notnull(ptr); - uint8_t * const CAST_FROM_VOIDP(dest, ptr); - const uint8_t * const CAST_FROM_VOIDP(src, mem); + uint8_t * CAST_FROM_VOIDP(dest, ptr); + const uint8_t * CAST_FROM_VOIDP(src, mem); if (pad_bytes == 0) { paranoid_invariant(aligned_memsize == mem_length); memcpy(dest, src, aligned_memsize); From 953d70f960421ef4a6a2d711b66ef3d04592efc7 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Sun, 10 Jun 2018 16:37:49 +0300 Subject: [PATCH 14/15] MDEV-15778: Remove packed attr from omt_ and subtree_ classes Undo the revert that happened by mystake in commit 7fca4b50ffbe750532cfcdb95bcd425ec1b8e22b. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 02f3f0d759a..6cf3f80d2bc 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} __attribute__((__packed__,aligned(4))); +} ; template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} __attribute__((__packed__)) ; +} ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} __attribute__((__packed__,aligned(4))); +} ; template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} __attribute__((__packed__,aligned(4))); +} ; } From 1d43f71c7b4c39a6fd196c104b5ebafb65376199 Mon Sep 17 00:00:00 2001 From: Rutuja Surve Date: Sun, 10 Jun 2018 11:19:39 +0300 Subject: [PATCH 15/15] MDEV-15021: mysqldump --tables --routines generates non importable dump file The order of outputting stored procedures is important. Stored procedures must be available on view creation, for views which make use of them. Make sure to print them before outputting tables. --- client/mysqldump.c | 12 ++++++------ mysql-test/r/mysqldump.result | 15 +++++++++++++++ mysql-test/t/mysqldump.test | 22 ++++++++++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/client/mysqldump.c b/client/mysqldump.c index 4e611ed5cb2..ad71a4dd0ce 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -4701,6 +4701,12 @@ static int dump_selected_tables(char *db, char **table_names, int tables) if (opt_xml) print_xml_tag(md_result_file, "", "\n", "database", "name=", db, NullS); + /* obtain dump of routines (procs/functions) */ + if (opt_routines && mysql_get_server_version(mysql) >= 50009) + { + DBUG_PRINT("info", ("Dumping routines for database %s", db)); + dump_routines_for_db(db); + } /* Dump each selected table */ for (pos= dump_tables; pos < end; pos++) { @@ -4729,12 +4735,6 @@ static int dump_selected_tables(char *db, char **table_names, int tables) DBUG_PRINT("info", ("Dumping events for database %s", db)); dump_events_for_db(db); } - /* obtain dump of routines (procs/functions) */ - if (opt_routines && mysql_get_server_version(mysql) >= 50009) - { - DBUG_PRINT("info", ("Dumping routines for database %s", db)); - dump_routines_for_db(db); - } free_root(&root, MYF(0)); my_free(order_by); order_by= 0; diff --git a/mysql-test/r/mysqldump.result b/mysql-test/r/mysqldump.result index 68de653742c..99b14189282 100644 --- a/mysql-test/r/mysqldump.result +++ b/mysql-test/r/mysqldump.result @@ -5401,3 +5401,18 @@ DELIMITER ; /*!50003 SET collation_connection = @saved_col_connection */ ; ALTER DATABASE `a\"'``b` CHARACTER SET utf8 COLLATE utf8_general_ci ; DROP DATABASE `a\"'``b`; +# +# MDEV-15021: Fix the order in which routines are called +# +use test; +CREATE FUNCTION f() RETURNS INT RETURN 1; +CREATE VIEW v1 AS SELECT f(); +# Running mysqldump -uroot test --routines --tables v1 > **vardir**/test.dmp +DROP VIEW v1; +DROP FUNCTION f; +# Running mysql -uroot test < **vardir**/test.dmp +# +# Cleanup after succesful import. +# +DROP VIEW v1; +DROP FUNCTION f; diff --git a/mysql-test/t/mysqldump.test b/mysql-test/t/mysqldump.test index 39ca5b3c58e..cb12919a823 100644 --- a/mysql-test/t/mysqldump.test +++ b/mysql-test/t/mysqldump.test @@ -2567,3 +2567,25 @@ let SEARCH_FILE=$MYSQLTEST_VARDIR/tmp/bug11505.sql; let SEARCH_PATTERN=Database: mysql; exec $MYSQL_DUMP mysql func > $SEARCH_FILE; source include/search_pattern_in_file.inc; + +--echo # +--echo # MDEV-15021: Fix the order in which routines are called +--echo # +use test; +CREATE FUNCTION f() RETURNS INT RETURN 1; +CREATE VIEW v1 AS SELECT f(); + +--echo # Running mysqldump -uroot test --routines --tables v1 > **vardir**/test.dmp +--exec $MYSQL_DUMP -uroot test --routines --tables v1 > $MYSQLTEST_VARDIR/test.dmp + +DROP VIEW v1; +DROP FUNCTION f; + +--echo # Running mysql -uroot test < **vardir**/test.dmp +--exec $MYSQL -uroot test < $MYSQLTEST_VARDIR/test.dmp + +--echo # +--echo # Cleanup after succesful import. +--echo # +DROP VIEW v1; +DROP FUNCTION f;