From 2da6f7cebae5a224f1f982981103a15a2ecda59a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 3 Nov 2014 15:43:44 +0200 Subject: [PATCH 01/36] MDEV-7017: Add function to print semaphore waits Add function to print to stderr all current semaphore waits. This function should be able to executed inside a gdb/ddd. --- storage/innobase/include/sync0arr.h | 6 +++ storage/innobase/include/sync0sync.h | 5 +++ storage/innobase/sync/sync0arr.c | 65 +++++++++++++++++++++++++++- storage/innobase/sync/sync0sync.c | 10 +++++ storage/xtradb/include/sync0arr.h | 6 +++ storage/xtradb/include/sync0sync.h | 5 +++ storage/xtradb/sync/sync0arr.c | 64 ++++++++++++++++++++++++++- storage/xtradb/sync/sync0sync.c | 10 +++++ 8 files changed, 169 insertions(+), 2 deletions(-) diff --git a/storage/innobase/include/sync0arr.h b/storage/innobase/include/sync0arr.h index 4bce9435577..a31c61a3806 100644 --- a/storage/innobase/include/sync0arr.h +++ b/storage/innobase/include/sync0arr.h @@ -138,6 +138,12 @@ sync_array_print_info( sync_array_t* arr); /*!< in: wait array */ +/**********************************************************************//** +Prints info of the wait array without using any mutexes/semaphores. */ +UNIV_INTERN +void +sync_array_print_innodb(void); + #ifndef UNIV_NONINL #include "sync0arr.ic" #endif diff --git a/storage/innobase/include/sync0sync.h b/storage/innobase/include/sync0sync.h index 9b07c4758c9..78f5b8dd02c 100644 --- a/storage/innobase/include/sync0sync.h +++ b/storage/innobase/include/sync0sync.h @@ -777,6 +777,11 @@ extern ut_list_base_node_t mutex_list; /** Mutex protecting the mutex_list variable */ extern mutex_t mutex_list_mutex; +/*******************************************************************//** +Get sync array */ +UNIV_INTERN +sync_array_t* +sync_array_get(void); #ifndef UNIV_NONINL #include "sync0sync.ic" diff --git a/storage/innobase/sync/sync0arr.c b/storage/innobase/sync/sync0arr.c index ba1d5cd8e61..5c4ba2058bd 100644 --- a/storage/innobase/sync/sync0arr.c +++ b/storage/innobase/sync/sync0arr.c @@ -2,7 +2,7 @@ Copyright (c) 1995, 2011, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2014, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1111,3 +1111,66 @@ sync_array_print_info( sync_array_exit(arr); } + +/**********************************************************************//** +Prints info of the wait array without using any mutexes/semaphores. */ +UNIV_INTERN +void +sync_array_print_innodb(void) +/*=========================*/ +{ + ulint i; + sync_array_t* arr = sync_array_get(); + + fputs("InnoDB: Semaphore wait debug output started for InnoDB:\n", stderr); + + for (i = 0; i < arr->n_cells; i++) { + void* wait_object; + sync_cell_t* cell; + os_thread_id_t reserver=(os_thread_id_t)ULINT_UNDEFINED; + ulint loop=0; + + cell = sync_array_get_nth_cell(arr, i); + + wait_object = cell->wait_object; + + if (wait_object == NULL || !cell->waiting) { + + continue; + } + + fputs("InnoDB: Warning: semaphore wait:\n", + stderr); + sync_array_cell_print(stderr, cell, &reserver); + + /* Try to output cell information for writer recursive way */ + while (reserver != (os_thread_id_t)ULINT_UNDEFINED) { + sync_cell_t* reserver_wait; + + reserver_wait = sync_array_find_thread(arr, reserver); + + if (reserver_wait && + reserver_wait->wait_object != NULL && + reserver_wait->waiting) { + fputs("InnoDB: Warning: Writer thread is waiting this semaphore:\n", + stderr); + sync_array_cell_print(stderr, reserver_wait, &reserver); + + if (reserver_wait->thread == reserver) { + reserver = (os_thread_id_t)ULINT_UNDEFINED; + } + } else { + reserver = (os_thread_id_t)ULINT_UNDEFINED; + } + + /* This is protection against loop */ + if (loop > 100) { + fputs("InnoDB: Warning: Too many waiting threads.\n", stderr); + break; + } + } + } + + fputs("InnoDB: Semaphore wait debug output ended:\n", stderr); + +} diff --git a/storage/innobase/sync/sync0sync.c b/storage/innobase/sync/sync0sync.c index 3ae77616faf..03cc3e46f3c 100644 --- a/storage/innobase/sync/sync0sync.c +++ b/storage/innobase/sync/sync0sync.c @@ -1665,3 +1665,13 @@ sync_print( sync_print_wait_info(file); } + +/*******************************************************************//** +Get sync array */ +UNIV_INTERN +sync_array_t* +sync_array_get(void) +/*================*/ +{ + return sync_primary_wait_array; +} diff --git a/storage/xtradb/include/sync0arr.h b/storage/xtradb/include/sync0arr.h index 4bce9435577..5fb0330aa10 100644 --- a/storage/xtradb/include/sync0arr.h +++ b/storage/xtradb/include/sync0arr.h @@ -138,6 +138,12 @@ sync_array_print_info( sync_array_t* arr); /*!< in: wait array */ +/**********************************************************************//** +Prints info of the wait array without using any mutexes/semaphores. */ +UNIV_INTERN +void +sync_array_print_xtradb(void); + #ifndef UNIV_NONINL #include "sync0arr.ic" #endif diff --git a/storage/xtradb/include/sync0sync.h b/storage/xtradb/include/sync0sync.h index b3b99b10630..8509058ab1f 100644 --- a/storage/xtradb/include/sync0sync.h +++ b/storage/xtradb/include/sync0sync.h @@ -793,6 +793,11 @@ extern ut_list_base_node_t mutex_list; /** Mutex protecting the mutex_list variable */ extern mutex_t mutex_list_mutex; +/*******************************************************************//** +Get sync array */ +UNIV_INTERN +sync_array_t* +sync_array_get(void); #ifndef UNIV_NONINL #include "sync0sync.ic" diff --git a/storage/xtradb/sync/sync0arr.c b/storage/xtradb/sync/sync0arr.c index 503cc38d0da..73cd6ea1739 100644 --- a/storage/xtradb/sync/sync0arr.c +++ b/storage/xtradb/sync/sync0arr.c @@ -2,7 +2,7 @@ Copyright (c) 1995, 2011, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2014, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1147,3 +1147,65 @@ sync_array_print_info( sync_array_exit(arr); } + +/**********************************************************************//** +Prints info of the wait array without using any mutexes/semaphores. */ +UNIV_INTERN +void +sync_array_print_xtradb(void) +/*=========================*/ +{ + ulint i; + sync_array_t* arr = sync_array_get(); + + fputs("InnoDB: Semaphore wait debug output started for XtraDB:\n", stderr); + + for (i = 0; i < arr->n_cells; i++) { + void* wait_object; + sync_cell_t* cell; + os_thread_id_t reserver=(os_thread_id_t)ULINT_UNDEFINED; + ulint loop=0; + + cell = sync_array_get_nth_cell(arr, i); + + wait_object = cell->wait_object; + + if (wait_object == NULL || !cell->waiting) { + + continue; + } + + fputs("InnoDB: Warning: semaphore wait:\n", + stderr); + sync_array_cell_print(stderr, cell, &reserver); + + /* Try to output cell information for writer recursive way */ + while (reserver != (os_thread_id_t)ULINT_UNDEFINED) { + sync_cell_t* reserver_wait; + + reserver_wait = sync_array_find_thread(arr, reserver); + + if (reserver_wait && + reserver_wait->wait_object != NULL && + reserver_wait->waiting) { + fputs("InnoDB: Warning: Writer thread is waiting this semaphore:\n", + stderr); + sync_array_cell_print(stderr, reserver_wait, &reserver); + + if (reserver_wait->thread == reserver) { + reserver = (os_thread_id_t)ULINT_UNDEFINED; + } + } else { + reserver = (os_thread_id_t)ULINT_UNDEFINED; + } + + /* This is protection against loop */ + if (loop > 100) { + fputs("InnoDB: Warning: Too many waiting threads.\n", stderr); + break; + } + } + } + + fputs("InnoDB: Semaphore wait debug output ended:\n", stderr); +} diff --git a/storage/xtradb/sync/sync0sync.c b/storage/xtradb/sync/sync0sync.c index 42803cdd20b..906a1d9b893 100644 --- a/storage/xtradb/sync/sync0sync.c +++ b/storage/xtradb/sync/sync0sync.c @@ -1644,3 +1644,13 @@ sync_print( sync_print_wait_info(file); } + +/*******************************************************************//** +Get sync array */ +UNIV_INTERN +sync_array_t* +sync_array_get(void) +/*================*/ +{ + return sync_primary_wait_array; +} From b46b469817a113da5d548d8431edff7bc576f6d2 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Mon, 3 Nov 2014 15:03:57 -0500 Subject: [PATCH 02/36] mark 5585.test as a big test --- mysql-test/suite/tokudb.bugs/t/5585.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/tokudb.bugs/t/5585.test b/mysql-test/suite/tokudb.bugs/t/5585.test index faf2dd48d80..98fc88e24d1 100644 --- a/mysql-test/suite/tokudb.bugs/t/5585.test +++ b/mysql-test/suite/tokudb.bugs/t/5585.test @@ -1,5 +1,6 @@ --source include/have_innodb.inc --source include/have_tokudb.inc +--source include/big_test.inc --disable_warnings drop table if exists t1, t2; From 7adf64d857e0ed2a048f4db2f6cc47e43ec21053 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Wed, 5 Nov 2014 04:15:43 -0500 Subject: [PATCH 03/36] DB-730 build tokudb without XA --- storage/tokudb/hatoku_hton.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index eb625231a19..903d15c7969 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -625,8 +625,8 @@ int tokudb_end(handlerton * hton, ha_panic_function type) { if (db_env) { if (tokudb_init_flags & DB_INIT_LOG) tokudb_cleanup_log_files(); -#if TOKU_INCLUDE_XA long total_prepared = 0; // count the total number of prepared txn's that we discard +#if TOKU_INCLUDE_XA while (1) { // get xid's const long n_xid = 1; From c14fdb70b11af23b45f15007a04a0b5ee2ceee7d Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Wed, 5 Nov 2014 11:24:27 -0500 Subject: [PATCH 04/36] DB-754 include my_config.h first for mariadb 5.5.40 --- storage/tokudb/ha_tokudb.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index f993fe84135..ce2ddf97d08 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -92,6 +92,7 @@ PATENT RIGHTS GRANT: #pragma implementation // gcc: Class implementation #endif +#include extern "C" { #include "stdint.h" #define __STDC_FORMAT_MACROS From 10532d81e39584eca94c059a495272d0f9eb86c3 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Fri, 7 Nov 2014 13:39:32 -0500 Subject: [PATCH 05/36] DB-742 set tokudb version for tokudb builds --- scripts/make.mysql.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/make.mysql.bash b/scripts/make.mysql.bash index 1bf258c5c3b..19ae233af99 100755 --- a/scripts/make.mysql.bash +++ b/scripts/make.mysql.bash @@ -155,6 +155,7 @@ function generate_cmake_cmd () { -D BUILD_CONFIG=mysql_release \ -D CMAKE_BUILD_TYPE=$cmake_build_type \ -D CMAKE_TOKUDB_REVISION=$ft_revision \ + -D TOKUDB_VERSION=tokudb-${tokudb_version} \ -D BUILD_TESTING=OFF \ -D USE_GTAGS=OFF \ -D USE_CTAGS=OFF \ @@ -212,8 +213,6 @@ if [ $build_tgz != 0 ] ; then mkdir -p build.$cmake_build_type pushd build.$cmake_build_type - export TOKUDB_VERSION=$tokudb_version - # actually build cmd=$(generate_cmake_cmd) if [ $? != 0 ] ; then exit 1; fi From 695ce3ad29a023f331f9be5534efb5d14af6bc1b Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Sat, 8 Nov 2014 08:59:55 -0500 Subject: [PATCH 06/36] DB-756 set cardinality data for partitioned tokudb tables --- .../tokudb.bugs/r/db756_card_part_hash.result | 21 ++++++++++++ .../r/db756_card_part_hash_1.result | 20 +++++++++++ .../r/db756_card_part_hash_1_pick.result | 21 ++++++++++++ .../r/db756_card_part_hash_2.result | 20 +++++++++++ .../r/db756_card_part_hash_2_pick.result | 21 ++++++++++++ .../tokudb.bugs/t/db756_card_part_hash.test | 15 +++++++++ .../tokudb.bugs/t/db756_card_part_hash_1.test | 14 ++++++++ .../t/db756_card_part_hash_1_pick.test | 15 +++++++++ .../tokudb.bugs/t/db756_card_part_hash_2.test | 14 ++++++++ .../t/db756_card_part_hash_2_pick.test | 15 +++++++++ storage/tokudb/ha_tokudb.cc | 33 +++++++------------ storage/tokudb/ha_tokudb.h | 3 ++ storage/tokudb/hatoku_defines.h | 5 ++- storage/tokudb/tokudb_card.h | 2 +- 14 files changed, 195 insertions(+), 24 deletions(-) create mode 100644 mysql-test/suite/tokudb.bugs/r/db756_card_part_hash.result create mode 100644 mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1.result create mode 100644 mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1_pick.result create mode 100644 mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2.result create mode 100644 mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2_pick.result create mode 100644 mysql-test/suite/tokudb.bugs/t/db756_card_part_hash.test create mode 100644 mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_1.test create mode 100644 mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_1_pick.test create mode 100644 mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_2.test create mode 100644 mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_2_pick.test diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash.result new file mode 100644 index 00000000000..c1ca24b0551 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash.result @@ -0,0 +1,21 @@ +set default_storage_engine='tokudb'; +drop table if exists t; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 2 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +insert into t values (1,1),(3,1),(5,1); +insert into t values (2,1),(4,1),(6,1); +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 6 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +analyze table t; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 6 NULL NULL BTREE +t 1 x 1 x A 2 NULL NULL YES BTREE +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1.result new file mode 100644 index 00000000000..c82c922a87a --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1.result @@ -0,0 +1,20 @@ +set default_storage_engine='tokudb'; +drop table if exists t; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 2 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +insert into t values (1,1),(3,1),(5,1); +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 4 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +analyze table t; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 4 NULL NULL BTREE +t 1 x 1 x A 1 NULL NULL YES BTREE +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1_pick.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1_pick.result new file mode 100644 index 00000000000..576d1adee6c --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1_pick.result @@ -0,0 +1,21 @@ +set default_storage_engine='tokudb'; +drop table if exists t; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 2 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +insert into t values (1,1),(3,2),(5,3); +insert into t values (2,1),(4,1),(6,1),(8,1); +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 7 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +analyze table t; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 7 NULL NULL BTREE +t 1 x 1 x A 1 NULL NULL YES BTREE +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2.result new file mode 100644 index 00000000000..8937ddabbe3 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2.result @@ -0,0 +1,20 @@ +set default_storage_engine='tokudb'; +drop table if exists t; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 2 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +insert into t values (2,1),(4,1),(6,1); +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 4 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +analyze table t; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 4 NULL NULL BTREE +t 1 x 1 x A 1 NULL NULL YES BTREE +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2_pick.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2_pick.result new file mode 100644 index 00000000000..89b51a5a6b0 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2_pick.result @@ -0,0 +1,21 @@ +set default_storage_engine='tokudb'; +drop table if exists t; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 2 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +insert into t values (1,1),(3,2),(5,3),(7,4); +insert into t values (2,1),(4,1),(6,1); +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 7 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +analyze table t; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 7 NULL NULL BTREE +t 1 x 1 x A 7 NULL NULL YES BTREE +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash.test b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash.test new file mode 100644 index 00000000000..97dda1cb1b8 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash.test @@ -0,0 +1,15 @@ +# DB-756 verify that cardinality is picked from SOME partition +source include/have_tokudb.inc; +source include/have_partition.inc; +set default_storage_engine='tokudb'; +disable_warnings; +drop table if exists t; +enable_warnings; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +insert into t values (1,1),(3,1),(5,1); +insert into t values (2,1),(4,1),(6,1); +show indexes from t; +analyze table t; +show indexes from t; +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_1.test b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_1.test new file mode 100644 index 00000000000..3ef66a4b1e6 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_1.test @@ -0,0 +1,14 @@ +# DB-756 verify that cardinality is picked from the partition where the rows are mapped. in this case, the last partition. +source include/have_tokudb.inc; +source include/have_partition.inc; +set default_storage_engine='tokudb'; +disable_warnings; +drop table if exists t; +enable_warnings; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +insert into t values (1,1),(3,1),(5,1); +show indexes from t; +analyze table t; +show indexes from t; +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_1_pick.test b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_1_pick.test new file mode 100644 index 00000000000..b8044641109 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_1_pick.test @@ -0,0 +1,15 @@ +# DB-756 verify that cardinality is chosen from the first partition +source include/have_tokudb.inc; +source include/have_partition.inc; +set default_storage_engine='tokudb'; +disable_warnings; +drop table if exists t; +enable_warnings; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +insert into t values (1,1),(3,2),(5,3); +insert into t values (2,1),(4,1),(6,1),(8,1); +show indexes from t; +analyze table t; +show indexes from t; +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_2.test b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_2.test new file mode 100644 index 00000000000..ca10218cb05 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_2.test @@ -0,0 +1,14 @@ +# DB-756 verify that cardinality is picked from the partition where the rows are mapped. in this case, the first partition. +source include/have_tokudb.inc; +source include/have_partition.inc; +set default_storage_engine='tokudb'; +disable_warnings; +drop table if exists t; +enable_warnings; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +insert into t values (2,1),(4,1),(6,1); +show indexes from t; +analyze table t; +show indexes from t; +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_2_pick.test b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_2_pick.test new file mode 100644 index 00000000000..2cc55ec864d --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/t/db756_card_part_hash_2_pick.test @@ -0,0 +1,15 @@ +# DB-756 verify that cardinality is chosen from the last partition +source include/have_tokudb.inc; +source include/have_partition.inc; +set default_storage_engine='tokudb'; +disable_warnings; +drop table if exists t; +enable_warnings; +create table t (id int, x int, primary key (id), key (x)) partition by hash(id) partitions 2; +show indexes from t; +insert into t values (1,1),(3,2),(5,3),(7,4); +insert into t values (2,1),(4,1),(6,1); +show indexes from t; +analyze table t; +show indexes from t; +drop table t; diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index ce2ddf97d08..a7b81319b5e 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -243,23 +243,20 @@ void TOKUDB_SHARE::destroy(void) { tokudb_pthread_mutex_destroy(&mutex); rwlock_destroy(&num_DBs_lock); tokudb_pthread_cond_destroy(&m_openclose_cond); + tokudb_my_free(rec_per_key); + rec_per_key = NULL; } // MUST have tokudb_mutex locked on input static TOKUDB_SHARE *get_share(const char *table_name, TABLE_SHARE* table_share) { TOKUDB_SHARE *share = NULL; int error = 0; - uint length; - - length = (uint) strlen(table_name); - + uint length = (uint) strlen(table_name); if (!(share = (TOKUDB_SHARE *) my_hash_search(&tokudb_open_tables, (uchar *) table_name, length))) { char *tmp_name; - // // create share and fill it with all zeroes // hence, all pointers are initialized to NULL - // share = (TOKUDB_SHARE *) tokudb_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &share, sizeof(*share), &tmp_name, length + 1, @@ -1596,11 +1593,7 @@ exit: return ret_val; } -int ha_tokudb::initialize_share( - const char* name, - int mode - ) -{ +int ha_tokudb::initialize_share(const char* name, int mode) { int error = 0; uint64_t num_rows = 0; DB_TXN* txn = NULL; @@ -1737,17 +1730,12 @@ int ha_tokudb::initialize_share( init_hidden_prim_key_info(txn); // initialize cardinality info from the status dictionary - { - uint total_key_parts = tokudb::compute_total_key_parts(table_share); - uint64_t rec_per_key[total_key_parts]; - error = tokudb::get_card_from_status(share->status_block, txn, total_key_parts, rec_per_key); - if (error == 0) { - tokudb::set_card_in_key_info(table, total_key_parts, rec_per_key); - } else { - for (uint i = 0; i < total_key_parts; i++) - rec_per_key[i] = 0; - tokudb::set_card_in_key_info(table, total_key_parts, rec_per_key); - } + share->n_rec_per_key = tokudb::compute_total_key_parts(table_share); + share->rec_per_key = (uint64_t *) tokudb_my_realloc(share->rec_per_key, share->n_rec_per_key * sizeof (uint64_t), MYF(MY_FAE)); + error = tokudb::get_card_from_status(share->status_block, txn, share->n_rec_per_key, share->rec_per_key); + if (error) { + for (uint i = 0; i < share->n_rec_per_key; i++) + share->rec_per_key[i] = 0; } error = 0; @@ -5943,6 +5931,7 @@ int ha_tokudb::info(uint flag) { } if ((flag & HA_STATUS_CONST)) { stats.max_data_file_length= 9223372036854775807ULL; + tokudb::set_card_in_key_info(table, share->n_rec_per_key, share->rec_per_key); } /* Don't return key if we got an error for the internal primary key */ diff --git a/storage/tokudb/ha_tokudb.h b/storage/tokudb/ha_tokudb.h index ba9c5e0d8ba..6c4055fc621 100644 --- a/storage/tokudb/ha_tokudb.h +++ b/storage/tokudb/ha_tokudb.h @@ -184,6 +184,9 @@ public: enum { CLOSED, OPENING, OPENED, CLOSING, ERROR } m_state; int m_error; int m_initialize_count; + + uint n_rec_per_key; + uint64_t *rec_per_key; }; typedef struct st_filter_key_part_info { diff --git a/storage/tokudb/hatoku_defines.h b/storage/tokudb/hatoku_defines.h index 91f717379d5..47ba0fe33f5 100644 --- a/storage/tokudb/hatoku_defines.h +++ b/storage/tokudb/hatoku_defines.h @@ -441,6 +441,8 @@ static inline void *tokudb_my_malloc(size_t s, myf flags) { } static inline void *tokudb_my_realloc(void *p, size_t s, myf flags) { + if (s == 0) + return p; #if 50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799 return my_realloc(0, p, s, flags); #else @@ -449,7 +451,8 @@ static inline void *tokudb_my_realloc(void *p, size_t s, myf flags) { } static inline void tokudb_my_free(void *ptr) { - my_free(ptr); + if (ptr) + my_free(ptr); } static inline char *tokudb_my_strdup(const char *p, myf flags) { diff --git a/storage/tokudb/tokudb_card.h b/storage/tokudb/tokudb_card.h index 4c9d7b3d353..a9439c124eb 100644 --- a/storage/tokudb/tokudb_card.h +++ b/storage/tokudb/tokudb_card.h @@ -217,7 +217,7 @@ namespace tokudb { // For each key part, put records per key part in *rec_per_key_part[key_part_index]. // Returns 0 if success, otherwise an error number. // TODO statistical dives into the FT - int analyze_card(DB *db, DB_TXN *txn, bool is_unique __attribute__((unused)), uint64_t num_key_parts, uint64_t *rec_per_key_part, + int analyze_card(DB *db, DB_TXN *txn, bool is_unique, uint64_t num_key_parts, uint64_t *rec_per_key_part, int (*key_compare)(DB *, const DBT *, const DBT *, uint), int (*analyze_progress)(void *extra, uint64_t rows), void *progress_extra) { int error = 0; From f127e2120c08f88550041d5a0d0f9c47f391f16c Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Sat, 8 Nov 2014 10:36:33 -0500 Subject: [PATCH 07/36] DB-757 compute cardinality when alter table analyze partition is run --- .../r/db757_part_alter_analyze.result | 50 +++++++++++++++++++ .../t/db757_part_alter_analyze.test | 26 ++++++++++ storage/tokudb/ha_tokudb_admin.cc | 2 +- 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/tokudb.bugs/r/db757_part_alter_analyze.result create mode 100644 mysql-test/suite/tokudb.bugs/t/db757_part_alter_analyze.test diff --git a/mysql-test/suite/tokudb.bugs/r/db757_part_alter_analyze.result b/mysql-test/suite/tokudb.bugs/r/db757_part_alter_analyze.result new file mode 100644 index 00000000000..d80e3e4eac8 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/r/db757_part_alter_analyze.result @@ -0,0 +1,50 @@ +set default_storage_engine='tokudb'; +drop table if exists t; +create table t (id int, x int, y int, primary key (id), key (x), key (y)) +partition by range(id) +( partition p0 values less than (10), partition p1 values less than maxvalue); +insert into t values (1,1,1),(2,1,2),(3,1,3),(4,1,4); +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 5 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 y 1 y A NULL NULL NULL YES BTREE +alter table t analyze partition p0; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 5 NULL NULL BTREE +t 1 x 1 x A 1 NULL NULL YES BTREE +t 1 y 1 y A 5 NULL NULL YES BTREE +alter table t analyze partition p1; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 5 NULL NULL BTREE +t 1 x 1 x A 1 NULL NULL YES BTREE +t 1 y 1 y A 5 NULL NULL YES BTREE +insert into t values (100,1,1),(200,2,1),(300,3,1),(400,4,1),(500,5,1); +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 9 NULL NULL BTREE +t 1 x 1 x A 2 NULL NULL YES BTREE +t 1 y 1 y A 9 NULL NULL YES BTREE +alter table t analyze partition p0; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 9 NULL NULL BTREE +t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 y 1 y A NULL NULL NULL YES BTREE +alter table t analyze partition p1; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 id A 9 NULL NULL BTREE +t 1 x 1 x A 9 NULL NULL YES BTREE +t 1 y 1 y A 1 NULL NULL YES BTREE +drop table t; diff --git a/mysql-test/suite/tokudb.bugs/t/db757_part_alter_analyze.test b/mysql-test/suite/tokudb.bugs/t/db757_part_alter_analyze.test new file mode 100644 index 00000000000..fc1599591be --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/t/db757_part_alter_analyze.test @@ -0,0 +1,26 @@ +source include/have_tokudb.inc; +source include/have_partition.inc; +set default_storage_engine='tokudb'; +disable_warnings; +drop table if exists t; +enable_warnings; + +create table t (id int, x int, y int, primary key (id), key (x), key (y)) +partition by range(id) +( partition p0 values less than (10), partition p1 values less than maxvalue); + +insert into t values (1,1,1),(2,1,2),(3,1,3),(4,1,4); +show indexes from t; +alter table t analyze partition p0; +show indexes from t; +alter table t analyze partition p1; +show indexes from t; + +insert into t values (100,1,1),(200,2,1),(300,3,1),(400,4,1),(500,5,1); +show indexes from t; +alter table t analyze partition p0; +show indexes from t; +alter table t analyze partition p1; +show indexes from t; + +drop table t; diff --git a/storage/tokudb/ha_tokudb_admin.cc b/storage/tokudb/ha_tokudb_admin.cc index 8fe17e1dfca..c3b895bf4f4 100644 --- a/storage/tokudb/ha_tokudb_admin.cc +++ b/storage/tokudb/ha_tokudb_admin.cc @@ -133,7 +133,7 @@ int ha_tokudb::analyze(THD *thd, HA_CHECK_OPT *check_opt) { int result = HA_ADMIN_OK; // stub out analyze if optimize is remapped to alter recreate + analyze - if (thd_sql_command(thd) != SQLCOM_ANALYZE) { + if (thd_sql_command(thd) != SQLCOM_ANALYZE && thd_sql_command(thd) != SQLCOM_ALTER_TABLE) { TOKUDB_HANDLER_DBUG_RETURN(result); } From b84a892fb2e6ee7fdedd04972e431a1457b7231f Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 10 Nov 2014 18:08:17 +0400 Subject: [PATCH 08/36] MDEV-7019 String::chop() is wrong and may potentially crash (MySQL bug#56492) Merging a fix from the upstream. --- sql/sql_string.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/sql_string.h b/sql/sql_string.h index 35398f284ee..1fce3ae6c6f 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -232,7 +232,9 @@ public: */ inline void chop() { - Ptr[str_length--]= '\0'; + str_length--; + Ptr[str_length]= '\0'; + DBUG_ASSERT(strlen(Ptr) == str_length); } inline void free() From 3180e76262f724ad83819c718af9a89fb490c78f Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 12 Nov 2014 11:10:13 +0100 Subject: [PATCH 09/36] MDEV-7089: Test failures in main.failed_auth_unixsocket and plugins.unix_socket depending on environment The test cases had some --replace_result $USER USER. The problem is that the value of $USER can be anything, depending on the name of the unix account that runs the test suite. So random parts of the result can be errorneously replaced, causing test failures. Fix by making the replacements more specific, so they will match only the intended stuff regardless of the value of $USER. --- mysql-test/suite/plugins/t/unix_socket.test | 10 +++++++--- mysql-test/t/failed_auth_unixsocket.test | 10 ++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/plugins/t/unix_socket.test b/mysql-test/suite/plugins/t/unix_socket.test index 2c1af9fb1da..1522c9b7cbe 100644 --- a/mysql-test/suite/plugins/t/unix_socket.test +++ b/mysql-test/suite/plugins/t/unix_socket.test @@ -12,11 +12,14 @@ eval install plugin unix_socket soname '$AUTH_SOCKET_SO'; --echo # with named user --echo # ---replace_result $USER USER +--let $replace=create user $USER +--replace_result $replace "create user USER" eval create user $USER identified via unix_socket; --write_file $MYSQLTEST_VARDIR/tmp/peercred_test.txt ---replace_result $USER USER +--let $replace1=$USER@localhost +--let $replace2=$USER@% +--replace_result $replace1 "USER@localhost" $replace2 "USER@%" select user(), current_user(), database(); EOF @@ -31,7 +34,8 @@ EOF --error 1 --exec $MYSQL_TEST -u foobar --plugin-dir=$plugindir < $MYSQLTEST_VARDIR/tmp/peercred_test.txt ---replace_result $USER USER +--let $replace=drop user $USER +--replace_result $replace "drop user USER" eval drop user $USER; --echo # diff --git a/mysql-test/t/failed_auth_unixsocket.test b/mysql-test/t/failed_auth_unixsocket.test index ba31cf6a59f..f7345f44698 100644 --- a/mysql-test/t/failed_auth_unixsocket.test +++ b/mysql-test/t/failed_auth_unixsocket.test @@ -16,11 +16,17 @@ change_user $USER; eval install plugin unix_socket soname '$AUTH_SOCKET_SO'; ---replace_result $MASTER_MYSOCK MASTER_SOCKET $MASTER_MYPORT MASTER_PORT $USER USER +# Make sure that the replace works, even if $USER is 'user' or something else +# that matches other parts of the error message. +--echo connect(localhost,USER,,test,MASTER_PORT,MASTER_SOCKET); +--let $replace=Access denied for user '$USER' +--replace_result $MASTER_MYSOCK MASTER_SOCKET $MASTER_MYPORT MASTER_PORT $replace "Access denied for user 'USER'" +--disable_query_log --error ER_ACCESS_DENIED_NO_PASSWORD_ERROR connect (fail,localhost,$USER); +--enable_query_log ---replace_result $USER USER +--replace_result $replace "Access denied for user 'USER'" --error ER_ACCESS_DENIED_NO_PASSWORD_ERROR change_user $USER; From 356451fca8fc9f38e0e3f9b43416f0383d621ba9 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 13 Nov 2014 10:04:45 +0100 Subject: [PATCH 10/36] MDEV-7103: Sporadic test falure in rpl.rpl_parallel_show_binlog_events_purge_logs The test case had a classic mistake: SET DEBUG_SYNC='now SIGNAL xxx' followed immediately by SET DEBUG_SYNC='RESET'. This makes it possible for the waiter to miss the signal, if it does not manage to wake up prior to the RESET. --- .../rpl/r/rpl_parallel_show_binlog_events_purge_logs.result | 1 - .../suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test | 1 - 2 files changed, 2 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_parallel_show_binlog_events_purge_logs.result b/mysql-test/suite/rpl/r/rpl_parallel_show_binlog_events_purge_logs.result index b69deb17c4c..d454fa41111 100644 --- a/mysql-test/suite/rpl/r/rpl_parallel_show_binlog_events_purge_logs.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_show_binlog_events_purge_logs.result @@ -7,7 +7,6 @@ SHOW BINLOG EVENTS; SET DEBUG_SYNC= 'now WAIT_FOR on_show_binlog_events'; FLUSH LOGS; SET DEBUG_SYNC= 'now SIGNAL end'; -SET DEBUG_SYNC= 'RESET'; [connection slave] SET DEBUG_SYNC= 'RESET'; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test b/mysql-test/suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test index 16d986268c9..83d847318d8 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test @@ -22,7 +22,6 @@ SET DEBUG_SYNC= 'after_show_binlog_events SIGNAL on_show_binlog_events WAIT_FOR SET DEBUG_SYNC= 'now WAIT_FOR on_show_binlog_events'; FLUSH LOGS; SET DEBUG_SYNC= 'now SIGNAL end'; -SET DEBUG_SYNC= 'RESET'; --echo [connection slave] --connection slave From 8c7ef99bb24f9faa2e71b97178482606d83934bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 13 Nov 2014 11:24:19 +0200 Subject: [PATCH 11/36] MDEV-7100: InnoDB error monitor might unnecessary wait log_sys mutex Analysis: InnoDB error monitor is responsible to call every second sync_arr_wake_threads_if_sema_free() to wake up possible hanging threads if they are missed in mutex_signal_object. This is not possible if error monitor itself is on mutex/semaphore wait. We should avoid all unnecessary mutex/semaphore waits on error monitor. Currently error monitor calls function buf_flush_stat_update() that calls log_get_lsn() function and there we will try to get log_sys mutex. Better, solution for error monitor is that in buf_flush_stat_update() we will try to get lsn with mutex_enter_nowait() and if we did not get mutex do not update the stats. Fix: Use log_get_lsn_nowait() function on buf_flush_stat_update() function. If returned lsn is 0, we do not update flush stats. log_get_lsn_nowait() will use mutex_enter_nowait() and if we get mutex we return a correct lsn if not we return 0. --- storage/innobase/buf/buf0flu.c | 10 +++++++++- storage/innobase/include/log0log.ic | 10 +++++----- storage/xtradb/buf/buf0flu.c | 10 +++++++++- storage/xtradb/include/log0log.ic | 10 +++++----- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/storage/innobase/buf/buf0flu.c b/storage/innobase/buf/buf0flu.c index 7abd014a364..da58d8392cf 100644 --- a/storage/innobase/buf/buf0flu.c +++ b/storage/innobase/buf/buf0flu.c @@ -2103,7 +2103,15 @@ buf_flush_stat_update(void) ib_uint64_t lsn; ulint n_flushed; - lsn = log_get_lsn(); + lsn = log_get_lsn_nowait(); + + /* log_get_lsn_nowait tries to get log_sys->mutex with + mutex_enter_nowait, if this does not succeed function + returns 0, do not use that value to update stats. */ + if (lsn == 0) { + return; + } + if (buf_flush_stat_cur.redo == 0) { /* First time around. Just update the current LSN and return. */ diff --git a/storage/innobase/include/log0log.ic b/storage/innobase/include/log0log.ic index a55bbeff789..8c8dc096a10 100644 --- a/storage/innobase/include/log0log.ic +++ b/storage/innobase/include/log0log.ic @@ -419,14 +419,14 @@ lsn_t log_get_lsn_nowait(void) /*=============*/ { - lsn_t lsn; + lsn_t lsn=0; - if (mutex_enter_nowait(&(log_sys->mutex))) - return 0; + if (!mutex_enter_nowait(&(log_sys->mutex))) { - lsn = log_sys->lsn; + lsn = log_sys->lsn; - mutex_exit(&(log_sys->mutex)); + mutex_exit(&(log_sys->mutex)); + } return(lsn); } diff --git a/storage/xtradb/buf/buf0flu.c b/storage/xtradb/buf/buf0flu.c index aa93053abae..43f6a4510a2 100644 --- a/storage/xtradb/buf/buf0flu.c +++ b/storage/xtradb/buf/buf0flu.c @@ -2214,7 +2214,15 @@ buf_flush_stat_update(void) ib_uint64_t lsn; ulint n_flushed; - lsn = log_get_lsn(); + lsn = log_get_lsn_nowait(); + + /* log_get_lsn_nowait tries to get log_sys->mutex with + mutex_enter_nowait, if this does not succeed function + returns 0, do not use that value to update stats. */ + if (lsn == 0) { + return; + } + if (buf_flush_stat_cur.redo == 0) { /* First time around. Just update the current LSN and return. */ diff --git a/storage/xtradb/include/log0log.ic b/storage/xtradb/include/log0log.ic index 82f1912c584..36bc5186c22 100644 --- a/storage/xtradb/include/log0log.ic +++ b/storage/xtradb/include/log0log.ic @@ -434,14 +434,14 @@ lsn_t log_get_lsn_nowait(void) /*=============*/ { - lsn_t lsn; + lsn_t lsn=0; - if (mutex_enter_nowait(&(log_sys->mutex))) - return 0; + if (!mutex_enter_nowait(&(log_sys->mutex))) { - lsn = log_sys->lsn; + lsn = log_sys->lsn; - mutex_exit(&(log_sys->mutex)); + mutex_exit(&(log_sys->mutex)); + } return(lsn); } From 06c7f493e35dd1d798f44b6920c4b7957d42c4de Mon Sep 17 00:00:00 2001 From: Sergey Petrunya Date: Thu, 13 Nov 2014 13:56:35 +0300 Subject: [PATCH 12/36] MDEV-7068: MRR accessing uninitialised bytes, test case failure main.innodb_mrr - Don't call index_reader->interrupt_read() if the index reader has returned all rows that matched its keys. --- sql/multi_range_read.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc index bbc287bf239..2479bc3258c 100644 --- a/sql/multi_range_read.cc +++ b/sql/multi_range_read.cc @@ -684,8 +684,19 @@ int Mrr_ordered_rndpos_reader::refill_from_index_reader() rowid_buffer->write_ptr2= (uchar*)&range_info; rowid_buffer->write(); } - - index_reader->interrupt_read(); + + /* + When index_reader_needs_refill=TRUE, this means we've got all of index + tuples for lookups keys that index_reader had. We are not in the middle + of an index read, so there is no need to call interrupt_read. + + Actually, we must not call interrupt_read(), because it could be that we + haven't read a single row (because all index lookups returned + HA_ERR_KEY_NOT_FOUND). In this case, interrupt_read() will cause [harmless] + valgrind warnings when trying to save garbage from table->record[0]. + */ + if (!index_reader_needs_refill) + index_reader->interrupt_read(); /* Sort the buffer contents by rowid */ rowid_buffer->sort((qsort2_cmp)rowid_cmp_reverse, (void*)file); From e7c356f7172cf0451eb4a1723623dfab54b8c3c3 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 15 Nov 2014 22:18:33 +0100 Subject: [PATCH 13/36] MDEV-6868: MariaDB server crash ( select with union and order by with subquery ) Excluding ORDER BY condition should be done after preparation it (even to catch syntax errors). --- mysql-test/r/union.result | 20 ++++++++++++++++++++ mysql-test/t/union.test | 20 ++++++++++++++++++++ sql/sql_union.cc | 24 ++++++++++++------------ 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/mysql-test/r/union.result b/mysql-test/r/union.result index 4ecac34d9fa..40f5a77e3d0 100644 --- a/mysql-test/r/union.result +++ b/mysql-test/r/union.result @@ -1935,3 +1935,23 @@ id select_type table type possible_keys key key_len ref rows Extra 3 UNION t1 ALL NULL NULL NULL NULL 4 NULL UNION RESULT ALL NULL NULL NULL NULL NULL drop table t1; +# +# MDEV-6868:MariaDB server crash ( select with union and order by +# with subquery ) +# +CREATE TABLE t1 ( id INTEGER, sample_name1 VARCHAR(100), sample_name2 VARCHAR(100), PRIMARY KEY(id) ); +INSERT INTO t1 ( id, sample_name1, sample_name2 ) VALUES ( 1, 'aaaa', 'bbbb' ), ( 2, 'cccc', 'dddd' ); +( +SELECT sample_name1 AS testname FROM t1 +) +UNION +( +SELECT sample_name2 AS testname FROM t1 C ORDER BY (SELECT T.sample_name1 FROM t1 T WHERE T.id = C.id) +) +; +testname +aaaa +cccc +bbbb +dddd +drop table t1; diff --git a/mysql-test/t/union.test b/mysql-test/t/union.test index a5d7dae606f..9204ddd22e5 100644 --- a/mysql-test/t/union.test +++ b/mysql-test/t/union.test @@ -1330,3 +1330,23 @@ create table t1 (a int); insert t1 values (1),(2),(3),(1); explain select 1 from dual where exists (select max(a) from t1 group by a union select a+2 from t1); drop table t1; + +--echo # +--echo # MDEV-6868:MariaDB server crash ( select with union and order by +--echo # with subquery ) +--echo # + +CREATE TABLE t1 ( id INTEGER, sample_name1 VARCHAR(100), sample_name2 VARCHAR(100), PRIMARY KEY(id) ); + +INSERT INTO t1 ( id, sample_name1, sample_name2 ) VALUES ( 1, 'aaaa', 'bbbb' ), ( 2, 'cccc', 'dddd' ); + +( + SELECT sample_name1 AS testname FROM t1 +) +UNION +( + SELECT sample_name2 AS testname FROM t1 C ORDER BY (SELECT T.sample_name1 FROM t1 T WHERE T.id = C.id) +) +; + +drop table t1; diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 1bbd2eb6c29..6e2e6e06ff7 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -315,18 +315,6 @@ bool st_select_lex_unit::prepare(THD *thd_arg, select_result *sel_result, can_skip_order_by= is_union_select && !(sl->braces && sl->explicit_limit); - /* - Remove all references from the select_lex_units to the subqueries that - are inside the ORDER BY clause. - */ - if (can_skip_order_by) - { - for (ORDER *ord= (ORDER *)sl->order_list.first; ord; ord= ord->next) - { - (*ord->item)->walk(&Item::eliminate_subselect_processor, FALSE, NULL); - } - } - saved_error= join->prepare(&sl->ref_pointer_array, sl->table_list.first, sl->with_wild, @@ -349,6 +337,18 @@ bool st_select_lex_unit::prepare(THD *thd_arg, select_result *sel_result, if (saved_error || (saved_error= thd_arg->is_fatal_error)) goto err; + /* + Remove all references from the select_lex_units to the subqueries that + are inside the ORDER BY clause. + */ + if (can_skip_order_by) + { + for (ORDER *ord= (ORDER *)sl->order_list.first; ord; ord= ord->next) + { + (*ord->item)->walk(&Item::eliminate_subselect_processor, FALSE, NULL); + } + } + /* Use items list of underlaid select for derived tables to preserve information about fields lengths and exact types From e072a647d9b98ffbaa606eeee9cad7cde82cd7f3 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 17 Nov 2014 17:24:04 +0400 Subject: [PATCH 14/36] MDEV-6865 Merge Bug#18935421 RPAD DIES WITH CERTAIN PADSTR INTPUTS.. --- mysql-test/r/ctype_utf16.result | 9 +++++++++ mysql-test/t/ctype_utf16.test | 6 ++++++ sql/item_strfunc.cc | 25 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result index 5d70764f3f8..bfb2d6a498c 100644 --- a/mysql-test/r/ctype_utf16.result +++ b/mysql-test/r/ctype_utf16.result @@ -1198,5 +1198,14 @@ a 512 Warnings: Warning 1260 Row 1 was cut by GROUP_CONCAT() # +# MDEV-6865 Merge Bug#18935421 RPAD DIES WITH CERTAIN PADSTR INTPUTS.. +# +DO RPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); +Warnings: +Warning 1300 Invalid utf16 character string: 'DE9899' +DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); +Warnings: +Warning 1300 Invalid utf16 character string: 'DE9899' +# # End of 5.5 tests # diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test index f42d30e1f00..3f9e5eece21 100644 --- a/mysql-test/t/ctype_utf16.test +++ b/mysql-test/t/ctype_utf16.test @@ -786,6 +786,12 @@ SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1; ## TODO: add tests for all engines # +--echo # +--echo # MDEV-6865 Merge Bug#18935421 RPAD DIES WITH CERTAIN PADSTR INTPUTS.. +--echo # +DO RPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); +DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); + --echo # --echo # End of 5.5 tests --echo # diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 66d980fb62d..e7e73794771 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -2721,6 +2721,19 @@ String *Item_func_rpad::val_str(String *str) res->set_charset(&my_charset_bin); rpad->set_charset(&my_charset_bin); } +#if MARIADB_VERSION_ID < 1000000 + /* + Well-formedness is handled on a higher level in 10.0, + no needs to check it here again. + */ + else + { + // This will chop off any trailing illegal characters from rpad. + String *well_formed_pad= args[2]->check_well_formed_result(rpad, false); + if (!well_formed_pad) + goto err; + } +#endif if (count <= (res_char_length= res->numchars())) { // String to pad is big enough @@ -2826,6 +2839,18 @@ String *Item_func_lpad::val_str(String *str) res->set_charset(&my_charset_bin); pad->set_charset(&my_charset_bin); } +#if MARIADB_VERSION_ID < 1000000 + /* + Well-formedness is handled on a higher level in 10.0, + no needs to check it here again. + */ else + { + // This will chop off any trailing illegal characters from pad. + String *well_formed_pad= args[2]->check_well_formed_result(pad, false); + if (!well_formed_pad) + goto err; + } +#endif res_char_length= res->numchars(); From 116baede5072587ba2a1c9585c427193b6d52339 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 17 Nov 2014 20:10:57 +0400 Subject: [PATCH 15/36] Sporadic failure in storage_engine/trx.xa_recovery test The test complains that the server failed to disappear upon shutdown / wait_for_disconnect. Trying to solve the probably race condition by adding a wait before restart. --- mysql-test/suite/storage_engine/trx/xa_recovery.test | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/storage_engine/trx/xa_recovery.test b/mysql-test/suite/storage_engine/trx/xa_recovery.test index 995878424f4..8256d068ade 100644 --- a/mysql-test/suite/storage_engine/trx/xa_recovery.test +++ b/mysql-test/suite/storage_engine/trx/xa_recovery.test @@ -56,10 +56,14 @@ XA PREPARE 'xa2'; --connection default --enable_reconnect --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -restart +wait EOF --shutdown_server 0 --source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF --source include/wait_until_connected_again.inc XA RECOVER; XA ROLLBACK 'xa1'; From b99328bbf8f46c1e2892b7912392727cbab7c476 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 17 Nov 2014 20:28:18 +0400 Subject: [PATCH 16/36] Re-enabling tests disabled due to MDEV-5266 and MySQL:65225 (fixed now) --- .../mysql-test/storage_engine/disabled.def | 1 - .../storage_engine/alter_table_online.rdiff | 44 +++++++++++++++++++ .../mysql-test/storage_engine/disabled.def | 2 - 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 storage/myisammrg/mysql-test/storage_engine/alter_table_online.rdiff delete mode 100644 storage/myisammrg/mysql-test/storage_engine/disabled.def diff --git a/storage/innobase/mysql-test/storage_engine/disabled.def b/storage/innobase/mysql-test/storage_engine/disabled.def index 3849170a7b8..b3b2c46f3f1 100644 --- a/storage/innobase/mysql-test/storage_engine/disabled.def +++ b/storage/innobase/mysql-test/storage_engine/disabled.def @@ -1,5 +1,4 @@ alter_table_online : MDEV-397 (Changing a column name via ALTER ONLINE does not work for InnoDB) -autoinc_vars : MySQL:65225 (InnoDB miscalculates auto-increment) tbl_opt_ai : MySQL:65901 (AUTO_INCREMENT option on InnoDB table is ignored if added before autoinc column) delete_low_prio : InnoDB does not use table-level locking insert_high_prio : InnoDB does not use table-level locking diff --git a/storage/myisammrg/mysql-test/storage_engine/alter_table_online.rdiff b/storage/myisammrg/mysql-test/storage_engine/alter_table_online.rdiff new file mode 100644 index 00000000000..c21211560e9 --- /dev/null +++ b/storage/myisammrg/mysql-test/storage_engine/alter_table_online.rdiff @@ -0,0 +1,44 @@ +--- suite/storage_engine/alter_table_online.result 2014-09-25 12:15:42.000000000 +0400 ++++ suite/storage_engine/alter_table_online.reject 2014-11-17 20:25:16.000000000 +0400 +@@ -9,20 +9,35 @@ + CREATE TEMPORARY TABLE t1 (a , b ) ENGINE= ; + INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'),(3,'c'); + ALTER ONLINE TABLE t1 MODIFY b DEFAULT 5; +-ERROR HY000: Can't execute the given 'ALTER' command as online ++# ERROR: Statement succeeded (expected results: ER_CANT_DO_ONLINE) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# Functionality or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + ALTER ONLINE TABLE t1 CHANGE b new_name ; +-ERROR HY000: Can't execute the given 'ALTER' command as online ++# ERROR: Statement succeeded (expected results: ER_CANT_DO_ONLINE) + ALTER ONLINE TABLE t1 COMMENT 'new comment'; +-ERROR HY000: Can't execute the given 'ALTER' command as online ++# ERROR: Statement succeeded (expected results: ER_CANT_DO_ONLINE) + ALTER ONLINE TABLE t1 RENAME TO t2; +-ERROR HY000: Can't execute the given 'ALTER' command as online ++# ERROR: Statement succeeded (expected results: ER_CANT_DO_ONLINE) + DROP TABLE t1; ++ERROR 42S02: Unknown table 't1' + CREATE TABLE t1 (a , b , c ) ENGINE= ; + INSERT INTO t1 (a,b,c) VALUES (1,100,'a'),(2,200,'b'),(3,300,'c'); + ALTER ONLINE TABLE t1 DROP COLUMN b, ADD b ; +-ERROR HY000: Can't execute the given 'ALTER' command as online ++# ERROR: Statement succeeded (expected results: ER_CANT_DO_ONLINE) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# Functionality or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + ALTER ONLINE TABLE t1 MODIFY b BIGINT ; +-ERROR HY000: Can't execute the given 'ALTER' command as online ++# ERROR: Statement succeeded (expected results: ER_CANT_DO_ONLINE) + ALTER ONLINE TABLE t1 ENGINE=MEMORY; + ERROR HY000: Can't execute the given 'ALTER' command as online + DROP TABLE t1; diff --git a/storage/myisammrg/mysql-test/storage_engine/disabled.def b/storage/myisammrg/mysql-test/storage_engine/disabled.def deleted file mode 100644 index 86b96c2ba2b..00000000000 --- a/storage/myisammrg/mysql-test/storage_engine/disabled.def +++ /dev/null @@ -1,2 +0,0 @@ -alter_table_online : MDEV-5266 (Temporary MERGE table with temporary underlying is broken by ALTER) - From 360c49c1b92ac0a7eb4895da376ec8d53f25fc72 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 8 Nov 2014 19:54:42 +0100 Subject: [PATCH 17/36] MDEV-6179: dynamic columns functions/cast()/convert() doesn't play nice with CREATE/ALTER TABLE When parsing a field declaration, grab type information from LEX before it's overwritten by further rules. Pass type information through the parser stack to the rule that needs it. --- mysql-test/r/create.result | 47 ++++++++++++++++++++++++++++++++++++++ mysql-test/t/create.test | 32 ++++++++++++++++++++++++++ sql/sql_lex.h | 9 ++++++++ sql/sql_yacc.yy | 31 ++++++++++++++----------- 4 files changed, 106 insertions(+), 13 deletions(-) diff --git a/mysql-test/r/create.result b/mysql-test/r/create.result index 751dd939361..d7023fea5a5 100644 --- a/mysql-test/r/create.result +++ b/mysql-test/r/create.result @@ -2430,3 +2430,50 @@ a b 1 1 unlock tables; drop table t1,t2; +# +# MDEV-6179: dynamic columns functions/cast()/convert() doesn't +# play nice with CREATE/ALTER TABLE +# +create table t1 ( +color char(32) as (COLUMN_GET(dynamic_cols, 1 as char)) persistent, +cl char(32) as (COLUMN_GET(COLUMN_ADD(COLUMN_CREATE(1 , 'blue' as char), 2, 'ttt'), i as char)) persistent, +item_name varchar(32) primary key, -- A common attribute for all items +i int, +dynamic_cols blob -- Dynamic columns will be stored here +); +INSERT INTO t1(item_name, dynamic_cols, i) VALUES +('MariaDB T-shirt', COLUMN_CREATE(1, 'blue', 2, 'XL'), 1); +INSERT INTO t1(item_name, dynamic_cols, i) VALUES +('Thinkpad Laptop', COLUMN_CREATE(1, 'black', 3, 500), 2); +select item_name, color, cl from t1; +item_name color cl +MariaDB T-shirt blue blue +Thinkpad Laptop black ttt +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `color` char(32) AS (COLUMN_GET(dynamic_cols, 1 as char)) PERSISTENT, + `cl` char(32) AS (COLUMN_GET(COLUMN_ADD(COLUMN_CREATE(1 , 'blue' as char), 2, 'ttt'), i as char)) PERSISTENT, + `item_name` varchar(32) NOT NULL, + `i` int(11) DEFAULT NULL, + `dynamic_cols` blob, + PRIMARY KEY (`item_name`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +drop table t1; +create table t1 ( +n int, +c char(32) as (convert(cast(n as char), char)) persistent +); +insert into t1(n) values (1),(2),(3); +select * from t1; +n c +1 1 +2 2 +3 3 +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `n` int(11) DEFAULT NULL, + `c` char(32) AS (convert(cast(n as char), char)) PERSISTENT +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +drop table t1; diff --git a/mysql-test/t/create.test b/mysql-test/t/create.test index 1fabb49138c..6de2c50ab36 100644 --- a/mysql-test/t/create.test +++ b/mysql-test/t/create.test @@ -2021,3 +2021,35 @@ connection default; select * from t1; unlock tables; drop table t1,t2; + +--echo # +--echo # MDEV-6179: dynamic columns functions/cast()/convert() doesn't +--echo # play nice with CREATE/ALTER TABLE +--echo # +create table t1 ( + color char(32) as (COLUMN_GET(dynamic_cols, 1 as char)) persistent, + cl char(32) as (COLUMN_GET(COLUMN_ADD(COLUMN_CREATE(1 , 'blue' as char), 2, 'ttt'), i as char)) persistent, + item_name varchar(32) primary key, -- A common attribute for all items + i int, + dynamic_cols blob -- Dynamic columns will be stored here +); +INSERT INTO t1(item_name, dynamic_cols, i) VALUES + ('MariaDB T-shirt', COLUMN_CREATE(1, 'blue', 2, 'XL'), 1); +INSERT INTO t1(item_name, dynamic_cols, i) VALUES + ('Thinkpad Laptop', COLUMN_CREATE(1, 'black', 3, 500), 2); + +select item_name, color, cl from t1; +show create table t1; + +drop table t1; + +create table t1 ( + n int, + c char(32) as (convert(cast(n as char), char)) persistent +); +insert into t1(n) values (1),(2),(3); + +select * from t1; +show create table t1; + +drop table t1; diff --git a/sql/sql_lex.h b/sql/sql_lex.h index e0bb5d11cbc..4f424000180 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -105,6 +105,15 @@ struct sys_var_with_base LEX_STRING base_name; }; +struct LEX_TYPE +{ + enum enum_field_types type; + char *length, *dec; + CHARSET_INFO *charset; + void set(int t, char *l, char *d, CHARSET_INFO *cs) + { type= (enum_field_types)t; length= l; dec= d; charset= cs; } +}; + #ifdef MYSQL_SERVER /* The following hack is needed because mysql_yacc.cc does not define diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 47bd3994fd3..777cef517c0 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -737,6 +737,7 @@ static bool add_create_index (LEX *lex, Key::Keytype type, LEX_STRING lex_str; LEX_STRING *lex_str_ptr; LEX_SYMBOL symbol; + LEX_TYPE lex_type; Table_ident *table; char *simple_string; Item *item; @@ -1471,13 +1472,15 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %type text_string opt_gconcat_separator +%type field_def + %type type type_with_opt_collate int_type real_type order_dir lock_option udf_type if_exists opt_local opt_table_options table_options table_option opt_if_not_exists opt_no_write_to_binlog opt_temporary all_or_any opt_distinct opt_ignore_leaves fulltext_options spatial_type union_option - start_transaction_opts field_def + start_transaction_opts union_opt select_derived_init option_type2 opt_natural_language_mode opt_query_expansion opt_ev_status opt_ev_on_completion ev_on_completion opt_ev_comment @@ -5483,11 +5486,11 @@ field_spec: field_def { LEX *lex=Lex; - if (add_field_to_list(lex->thd, &$1, (enum enum_field_types) $3, - lex->length,lex->dec,lex->type, + if (add_field_to_list(lex->thd, &$1, $3.type, + $3.length, $3.dec, lex->type, lex->default_value, lex->on_update_value, &lex->comment, - lex->change,&lex->interval_list,lex->charset, + lex->change, &lex->interval_list, $3.charset, lex->uint_geom_type, lex->vcol_info, lex->option_list)) MYSQL_YYABORT; @@ -5495,13 +5498,15 @@ field_spec: ; field_def: - type opt_attribute {} - | type opt_generated_always AS '(' virtual_column_func ')' - vcol_opt_specifier - vcol_opt_attribute + type opt_attribute + { $$.set($1, Lex->length, Lex->dec, Lex->charset); } + | type opt_generated_always AS + { $$.set($1, Lex->length, Lex->dec, Lex->charset); } + '(' virtual_column_func ')' vcol_opt_specifier vcol_opt_attribute { - $$= (enum enum_field_types)MYSQL_TYPE_VIRTUAL; - Lex->vcol_info->set_field_type((enum enum_field_types) $1); + $$= $4; + Lex->vcol_info->set_field_type($$.type); + $$.type= (enum enum_field_types)MYSQL_TYPE_VIRTUAL; } ; @@ -6892,11 +6897,11 @@ alter_list_item: { LEX *lex=Lex; if (add_field_to_list(lex->thd,&$3, - (enum enum_field_types) $5, - lex->length,lex->dec,lex->type, + $5.type, + $5.length, $5.dec, lex->type, lex->default_value, lex->on_update_value, &lex->comment, - $3.str, &lex->interval_list, lex->charset, + $3.str, &lex->interval_list, $5.charset, lex->uint_geom_type, lex->vcol_info, lex->option_list)) MYSQL_YYABORT; From c4598ce30094a084dec7cd4aba157e1a94d488d6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 10 Nov 2014 19:17:39 +0100 Subject: [PATCH 18/36] MDEV-6862 "#error " and third-party libraries only enforce the include order if SAFE_MUTEX is defined (that is, in MariaDB source builds in debug mode) --- config.h.cmake | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/config.h.cmake b/config.h.cmake index 4b0769d5cfa..34f77dba050 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -648,9 +648,13 @@ included first (or at least before - so, practically, before including any system headers). - __GLIBC__ is defined in + Check the include order by looking at __GLIBC__ (defined in ) + + But we cannot force all third-party clients/connectors to include + my_config.h first. So, their crashes are their responsibility, + we enable this check only for MariaDB sources (SAFE_MUTEX check). */ -#ifdef __GLIBC__ +#if defined(__GLIBC__) && defined(SAFE_MUTEX) #error MUST be included first! #endif From 43c176a7791c3d9c373cc43f28e78709611c0cd8 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Mon, 10 Nov 2014 16:34:55 -0500 Subject: [PATCH 19/36] speed up tokudb handler unit tests --- storage/tokudb/tests/Makefile | 7 +- .../{tokudb_math_test.cc => math_test_int.cc} | 125 +--------- storage/tokudb/tests/math_test_uint.cc | 214 ++++++++++++++++++ storage/tokudb/tests/vlq_test.cc | 100 +------- storage/tokudb/tests/vlq_test_uint32.cc | 159 +++++++++++++ storage/tokudb/tests/vlq_test_uint64.cc | 161 +++++++++++++ 6 files changed, 548 insertions(+), 218 deletions(-) rename storage/tokudb/tests/{tokudb_math_test.cc => math_test_int.cc} (71%) create mode 100644 storage/tokudb/tests/math_test_uint.cc create mode 100644 storage/tokudb/tests/vlq_test_uint32.cc create mode 100644 storage/tokudb/tests/vlq_test_uint64.cc diff --git a/storage/tokudb/tests/Makefile b/storage/tokudb/tests/Makefile index a565c213fff..309d3ec6af0 100644 --- a/storage/tokudb/tests/Makefile +++ b/storage/tokudb/tests/Makefile @@ -2,7 +2,7 @@ SRCS = $(wildcard *.cc) TARGETS = $(patsubst %.cc,%,$(SRCS)) CHECKS = $(patsubst %,%.check,$(TARGETS)) CPPFLAGS = -I.. -D__STDC_FORMAT_MACROS -CXXFLAGS = -g -Wall -Wextra -Wno-missing-field-initializers -Wshadow +CXXFLAGS = -g -Wall -Wextra -Wno-missing-field-initializers -Wshadow -fopenmp FRACTALTREE_BASE_DIR = ../ft-index FRACTALTREE_INSTALL_DIR = $(FRACTALTREE_BASE_DIR)/install.debug @@ -20,12 +20,15 @@ clean: check: $(CHECKS) true -%.check: % +%.check: % LD_LIBRARY_PATH=$(FRACTALTREE_INSTALL_DIR)/lib $(VALGRIND) ./$< card.check: card_test.check card_1.check card_inf.check card_inf_1.check card_random_1.check card_etime.check true +vlq.check: vlq_test.check vlq_test_uint32.check vlq_test_uint64.check + true + max_test.check: max_test $(VALGRIND) ./$< 1 2 diff --git a/storage/tokudb/tests/tokudb_math_test.cc b/storage/tokudb/tests/math_test_int.cc similarity index 71% rename from storage/tokudb/tests/tokudb_math_test.cc rename to storage/tokudb/tests/math_test_int.cc index 0c4be8c1242..1121c9f4a9d 100644 --- a/storage/tokudb/tests/tokudb_math_test.cc +++ b/storage/tokudb/tests/math_test_int.cc @@ -96,113 +96,6 @@ PATENT RIGHTS GRANT: #include using namespace tokudb; -static void test_uint_range(uint length_bits) { - assert(uint_low_endpoint(length_bits) == 0); - if (length_bits == 64) - assert(uint_high_endpoint(length_bits) == ~0ULL); - else - assert(uint_high_endpoint(length_bits) == (1ULL< (1ULL<<8)-1) - assert(over); - else - assert(!over && n == (m % 256)); - n = uint_sub(x, y, 8, &over); - m = x - y; - if (m > x) - assert(over); - else - assert(!over && n == (m % 256)); - } - } -} - -static void test_uint16() { - printf("%s\n", __FUNCTION__); - test_uint_range(16); - bool over; - uint16_t n; - uint64_t m; - for (uint64_t x = 0; x <= (1ULL<<16)-1; x++) { - for (uint64_t y = 0; y <= (1ULL<<16)-1; y++) { - n = uint_add(x, y, 16, &over); - m = x + y; - if (m > (1ULL<<16)-1) - assert(over); - else - assert(!over && n == (m % (1ULL<<16))); - n = uint_sub(x, y, 16, &over); - m = x - y; - if (m > x) - assert(over); - else - assert(!over && n == (m % (1ULL<<16))); - } - } -} - -static void test_uint24() { - printf("%s\n", __FUNCTION__); - test_uint_range(24); - bool over; - uint64_t s; - - s = uint_add((1ULL<<24)-1, (1ULL<<24)-1, 24, &over); assert(over); - s = uint_add((1ULL<<24)-1, 1, 24, &over); assert(over); - s = uint_add((1ULL<<24)-1, 0, 24, &over); assert(!over && s == (1ULL<<24)-1); - s = uint_add(0, 1, 24, &over); assert(!over && s == 1); - s = uint_add(0, 0, 24, &over); assert(!over && s == 0); - s = uint_sub(0, 0, 24, &over); assert(!over && s == 0); - s = uint_sub(0, 1, 24, &over); assert(over); - s = uint_sub(0, (1ULL<<24)-1, 24, &over); assert(over); - s = uint_sub((1ULL<<24)-1, (1ULL<<24)-1, 24, &over); assert(!over && s == 0); -} - -static void test_uint32() { - printf("%s\n", __FUNCTION__); - test_uint_range(32); - bool over; - uint64_t s; - - s = uint_add((1ULL<<32)-1, (1ULL<<32)-1, 32, &over); assert(over); - s = uint_add((1ULL<<32)-1, 1, 32, &over); assert(over); - s = uint_add((1ULL<<32)-1, 0, 32, &over); assert(!over && s == (1ULL<<32)-1); - s = uint_add(0, 1, 32, &over); assert(!over && s == 1); - s = uint_add(0, 0, 32, &over); assert(!over && s == 0); - s = uint_sub(0, 0, 32, &over); assert(!over && s == 0); - s = uint_sub(0, 1, 32, &over); assert(over); - s = uint_sub(0, (1ULL<<32)-1, 32, &over); assert(over); - s = uint_sub((1ULL<<32)-1, (1ULL<<32)-1, 32, &over); assert(!over && s == 0); -} - -static void test_uint64() { - printf("%s\n", __FUNCTION__); - test_uint_range(64); - bool over; - uint64_t s; - - s = uint_add(~0ULL, ~0ULL, 64, &over); assert(over); - s = uint_add(~0ULL, 1, 64, &over); assert(over); - s = uint_add(~0ULL, 0, 64, &over); assert(!over && s == ~0ULL); - s = uint_add(0, 1, 64, &over); assert(!over && s == 1); - s = uint_add(0, 0, 64, &over); assert(!over && s == 0); - s = uint_sub(0, 0, 64, &over); assert(!over && s == 0); - s = uint_sub(0, 1, 64, &over); assert(over); - s = uint_sub(0, ~0ULL, 64, &over); assert(over); - s = uint_sub(~0ULL, ~0ULL, 64, &over); assert(!over && s == 0); -} - static int64_t sign_extend(uint length_bits, int64_t n) { return n | ~((1ULL<<(length_bits-1))-1); } @@ -359,18 +252,12 @@ static void test_int_sign() { } int main() { - if (1) test_int_sign(); - if (1) test_int8(); - if (1) test_int16(); - if (1) test_int24(); - if (1) test_int32(); - if (1) test_int64(); - - if (1) test_uint8(); - if (1) test_uint16(); - if (1) test_uint24(); - if (1) test_uint32(); - if (1) test_uint64(); + test_int_sign(); + test_int8(); + test_int16(); + test_int24(); + test_int32(); + test_int64(); return 0; } diff --git a/storage/tokudb/tests/math_test_uint.cc b/storage/tokudb/tests/math_test_uint.cc new file mode 100644 index 00000000000..b44dc83c146 --- /dev/null +++ b/storage/tokudb/tests/math_test_uint.cc @@ -0,0 +1,214 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." +#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it." +#include +#include +#include +#include +#include +#include +using namespace tokudb; + +static void test_uint_range(uint length_bits) { + assert(uint_low_endpoint(length_bits) == 0); + if (length_bits == 64) + assert(uint_high_endpoint(length_bits) == ~0ULL); + else + assert(uint_high_endpoint(length_bits) == (1ULL< (1ULL<<8)-1) + assert(over); + else + assert(!over && n == (m % 256)); + n = uint_sub(x, y, 8, &over); + m = x - y; + if (m > x) + assert(over); + else + assert(!over && n == (m % 256)); + } + } +} + +static void test_uint16() { + printf("%s\n", __FUNCTION__); + test_uint_range(16); + bool over; + uint16_t n; + uint64_t m; + for (uint64_t x = 0; x <= (1ULL<<16)-1; x++) { + for (uint64_t y = 0; y <= (1ULL<<16)-1; y++) { + n = uint_add(x, y, 16, &over); + m = x + y; + if (m > (1ULL<<16)-1) + assert(over); + else + assert(!over && n == (m % (1ULL<<16))); + n = uint_sub(x, y, 16, &over); + m = x - y; + if (m > x) + assert(over); + else + assert(!over && n == (m % (1ULL<<16))); + } + } +} + +static void test_uint24() { + printf("%s\n", __FUNCTION__); + test_uint_range(24); + bool over; + uint64_t s; + + s = uint_add((1ULL<<24)-1, (1ULL<<24)-1, 24, &over); assert(over); + s = uint_add((1ULL<<24)-1, 1, 24, &over); assert(over); + s = uint_add((1ULL<<24)-1, 0, 24, &over); assert(!over && s == (1ULL<<24)-1); + s = uint_add(0, 1, 24, &over); assert(!over && s == 1); + s = uint_add(0, 0, 24, &over); assert(!over && s == 0); + s = uint_sub(0, 0, 24, &over); assert(!over && s == 0); + s = uint_sub(0, 1, 24, &over); assert(over); + s = uint_sub(0, (1ULL<<24)-1, 24, &over); assert(over); + s = uint_sub((1ULL<<24)-1, (1ULL<<24)-1, 24, &over); assert(!over && s == 0); +} + +static void test_uint32() { + printf("%s\n", __FUNCTION__); + test_uint_range(32); + bool over; + uint64_t s; + + s = uint_add((1ULL<<32)-1, (1ULL<<32)-1, 32, &over); assert(over); + s = uint_add((1ULL<<32)-1, 1, 32, &over); assert(over); + s = uint_add((1ULL<<32)-1, 0, 32, &over); assert(!over && s == (1ULL<<32)-1); + s = uint_add(0, 1, 32, &over); assert(!over && s == 1); + s = uint_add(0, 0, 32, &over); assert(!over && s == 0); + s = uint_sub(0, 0, 32, &over); assert(!over && s == 0); + s = uint_sub(0, 1, 32, &over); assert(over); + s = uint_sub(0, (1ULL<<32)-1, 32, &over); assert(over); + s = uint_sub((1ULL<<32)-1, (1ULL<<32)-1, 32, &over); assert(!over && s == 0); +} + +static void test_uint64() { + printf("%s\n", __FUNCTION__); + test_uint_range(64); + bool over; + uint64_t s; + + s = uint_add(~0ULL, ~0ULL, 64, &over); assert(over); + s = uint_add(~0ULL, 1, 64, &over); assert(over); + s = uint_add(~0ULL, 0, 64, &over); assert(!over && s == ~0ULL); + s = uint_add(0, 1, 64, &over); assert(!over && s == 1); + s = uint_add(0, 0, 64, &over); assert(!over && s == 0); + s = uint_sub(0, 0, 64, &over); assert(!over && s == 0); + s = uint_sub(0, 1, 64, &over); assert(over); + s = uint_sub(0, ~0ULL, 64, &over); assert(over); + s = uint_sub(~0ULL, ~0ULL, 64, &over); assert(!over && s == 0); +} + +int main() { + test_uint8(); + test_uint16(); + test_uint24(); + test_uint32(); + test_uint64(); + return 0; +} + diff --git a/storage/tokudb/tests/vlq_test.cc b/storage/tokudb/tests/vlq_test.cc index 7e47d85684e..7cd17dda15f 100644 --- a/storage/tokudb/tests/vlq_test.cc +++ b/storage/tokudb/tests/vlq_test.cc @@ -120,98 +120,6 @@ static void test_vlq_uint32_error(void) { assert(in_s == 2 && n == 128); } -static void test_vlq_uint32(void) { - uint32_t n; - unsigned char b[5]; - size_t out_s, in_s; - - printf("%u\n", 0); - for (uint32_t v = 0; v < (1<<7); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 1); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 1 && n == v); - } - - printf("%u\n", 1<<7); - for (uint32_t v = (1<<7); v < (1<<14); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 2); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 2 && n == v); - } - - printf("%u\n", 1<<14); - for (uint32_t v = (1<<14); v < (1<<21); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 3); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 3 && n == v); - } - - printf("%u\n", 1<<21); - for (uint32_t v = (1<<21); v < (1<<28); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 4); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 4 && n == v); - } - - printf("%u\n", 1<<28); - for (uint32_t v = (1<<28); v != 0; v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 5); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 5 && n == v); - } -} - -static void test_vlq_uint64(void) { - uint64_t n; - unsigned char b[10]; - size_t out_s, in_s; - - printf("%u\n", 0); - for (uint64_t v = 0; v < (1<<7); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 1); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 1 && n == v); - } - - printf("%u\n", 1<<7); - for (uint64_t v = (1<<7); v < (1<<14); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 2); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 2 && n == v); - } - - printf("%u\n", 1<<14); - for (uint64_t v = (1<<14); v < (1<<21); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 3); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 3 && n == v); - } - - printf("%u\n", 1<<21); - for (uint64_t v = (1<<21); v < (1<<28); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 4); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 4 && n == v); - } - - printf("%u\n", 1<<28); - for (uint64_t v = (1<<28); v < (1ULL<<35); v++) { - out_s = tokudb::vlq_encode_ui(v, b, sizeof b); - assert(out_s == 5); - in_s = tokudb::vlq_decode_ui(&n, b, out_s); - assert(in_s == 5 && n == v); - } -} - static void test_80000000(void) { uint64_t n; unsigned char b[10]; @@ -235,10 +143,8 @@ static void test_100000000(void) { } int main(void) { - if (1) test_vlq_uint32_error(); - if (1) test_80000000(); - if (1) test_100000000(); - if (1) test_vlq_uint32(); - if (1) test_vlq_uint64(); + test_vlq_uint32_error(); + test_80000000(); + test_100000000(); return 0; } diff --git a/storage/tokudb/tests/vlq_test_uint32.cc b/storage/tokudb/tests/vlq_test_uint32.cc new file mode 100644 index 00000000000..494d3155389 --- /dev/null +++ b/storage/tokudb/tests/vlq_test_uint32.cc @@ -0,0 +1,159 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." +#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it." +#include +#include +#include +#include +#include + +namespace tokudb { + template size_t vlq_encode_ui(uint32_t n, void *p, size_t s); + template size_t vlq_decode_ui(uint32_t *np, void *p, size_t s); + template size_t vlq_encode_ui(uint64_t n, void *p, size_t s); + template size_t vlq_decode_ui(uint64_t *np, void *p, size_t s); +}; + +static void test_vlq_uint32(void) { + printf("%u\n", 0); + for (uint32_t v = 0; v < (1<<7); v++) { + unsigned char b[5]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 1); + uint32_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 1 && n == v); + } + + printf("%u\n", 1<<7); + for (uint32_t v = (1<<7); v < (1<<14); v++) { + unsigned char b[5]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 2); + uint32_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 2 && n == v); + } + + printf("%u\n", 1<<14); + for (uint32_t v = (1<<14); v < (1<<21); v++) { + unsigned char b[5]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 3); + uint32_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 3 && n == v); + } + + printf("%u\n", 1<<21); + for (uint32_t v = (1<<21); v < (1<<28); v++) { + unsigned char b[5]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 4); + uint32_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 4 && n == v); + } + + printf("%u\n", 1<<28); + for (uint32_t v = (1<<28); v != 0; v++) { + unsigned char b[5]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 5); + uint32_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 5 && n == v); + } +} + +int main(void) { + test_vlq_uint32(); + return 0; +} diff --git a/storage/tokudb/tests/vlq_test_uint64.cc b/storage/tokudb/tests/vlq_test_uint64.cc new file mode 100644 index 00000000000..d29636fd6be --- /dev/null +++ b/storage/tokudb/tests/vlq_test_uint64.cc @@ -0,0 +1,161 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." +#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it." +#include +#include +#include +#include +#include + +namespace tokudb { + template size_t vlq_encode_ui(uint32_t n, void *p, size_t s); + template size_t vlq_decode_ui(uint32_t *np, void *p, size_t s); + template size_t vlq_encode_ui(uint64_t n, void *p, size_t s); + template size_t vlq_decode_ui(uint64_t *np, void *p, size_t s); +}; + +static void test_vlq_uint64(void) { + printf("%u\n", 0); + for (uint64_t v = 0; v < (1<<7); v++) { + unsigned char b[10]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 1); + uint64_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 1 && n == v); + } + + printf("%u\n", 1<<7); + for (uint64_t v = (1<<7); v < (1<<14); v++) { + unsigned char b[10]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 2); + uint64_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 2 && n == v); + } + + printf("%u\n", 1<<14); + for (uint64_t v = (1<<14); v < (1<<21); v++) { + unsigned char b[10]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 3); + uint64_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 3 && n == v); + } + + printf("%u\n", 1<<21); + for (uint64_t v = (1<<21); v < (1<<28); v++) { + unsigned char b[10]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 4); + uint64_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 4 && n == v); + } + + printf("%u\n", 1<<28); +#pragma omp parallel num_threads(4) +#pragma omp for + for (uint64_t v = (1<<28); v < (1ULL<<35); v++) { + unsigned char b[10]; + size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); + assert(out_s == 5); + uint64_t n; + size_t in_s = tokudb::vlq_decode_ui(&n, b, out_s); + assert(in_s == 5 && n == v); + } +} + +int main(void) { + test_vlq_uint64(); + return 0; +} From fa92c90393badcb165412da935d6a10d4b24a684 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Wed, 12 Nov 2014 08:38:04 -0500 Subject: [PATCH 20/36] speed up tokudb handler unit tests --- storage/tokudb/tests/Makefile | 20 ++++++++++++++++---- storage/tokudb/tests/vlq_test_uint64.cc | 25 +++++++++++++++++-------- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/storage/tokudb/tests/Makefile b/storage/tokudb/tests/Makefile index 309d3ec6af0..1943cc01867 100644 --- a/storage/tokudb/tests/Makefile +++ b/storage/tokudb/tests/Makefile @@ -2,7 +2,11 @@ SRCS = $(wildcard *.cc) TARGETS = $(patsubst %.cc,%,$(SRCS)) CHECKS = $(patsubst %,%.check,$(TARGETS)) CPPFLAGS = -I.. -D__STDC_FORMAT_MACROS -CXXFLAGS = -g -Wall -Wextra -Wno-missing-field-initializers -Wshadow -fopenmp +CXXFLAGS = -g -Wall -Wextra -Wno-missing-field-initializers -Wshadow +ifdef USE_OPENMP +CPPFLAGS += -DUSE_OPENMP +CXXFLAGS += -fopenmp +endif FRACTALTREE_BASE_DIR = ../ft-index FRACTALTREE_INSTALL_DIR = $(FRACTALTREE_BASE_DIR)/install.debug @@ -17,21 +21,29 @@ all: $(TARGETS) clean: rm -rf $(TARGETS) *.gcov *.gcno *.gcda *.testdir *.dSYM -check: $(CHECKS) - true - %.check: % LD_LIBRARY_PATH=$(FRACTALTREE_INSTALL_DIR)/lib $(VALGRIND) ./$< card.check: card_test.check card_1.check card_inf.check card_inf_1.check card_random_1.check card_etime.check true +ifndef USE_OPENMP +# unravel vlq_test_uint64 8 times +vlq_test_uint64_%.check: + LD_LIBRARY_PATH=$(FRACTALTREE_INSTALL_DIR)/lib $(VALGRIND) ./vlq_test_uint64 $(patsubst vlq_test_uint64_%.check,%,$@) 8 +vlq_test_uint64.check: $(foreach i,0 1 2 3 4 5 6 7,vlq_test_uint64_$(i).check) + true +endif + vlq.check: vlq_test.check vlq_test_uint32.check vlq_test_uint64.check true max_test.check: max_test $(VALGRIND) ./$< 1 2 +check: $(CHECKS) + true + %: %.cc $(CXX) $(CPPFLAGS) $(CXXFLAGS) -g -o $@ $< diff --git a/storage/tokudb/tests/vlq_test_uint64.cc b/storage/tokudb/tests/vlq_test_uint64.cc index d29636fd6be..79b83a9427a 100644 --- a/storage/tokudb/tests/vlq_test_uint64.cc +++ b/storage/tokudb/tests/vlq_test_uint64.cc @@ -101,9 +101,11 @@ namespace tokudb { template size_t vlq_decode_ui(uint64_t *np, void *p, size_t s); }; -static void test_vlq_uint64(void) { +// test a slice of the number space where the slice is described by +// a start number and a stride through the space. +static void test_vlq_uint64(uint64_t start, uint64_t stride) { printf("%u\n", 0); - for (uint64_t v = 0; v < (1<<7); v++) { + for (uint64_t v = 0 + start; v < (1<<7); v += stride) { unsigned char b[10]; size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); assert(out_s == 1); @@ -113,7 +115,7 @@ static void test_vlq_uint64(void) { } printf("%u\n", 1<<7); - for (uint64_t v = (1<<7); v < (1<<14); v++) { + for (uint64_t v = (1<<7) + start; v < (1<<14); v += stride) { unsigned char b[10]; size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); assert(out_s == 2); @@ -123,7 +125,7 @@ static void test_vlq_uint64(void) { } printf("%u\n", 1<<14); - for (uint64_t v = (1<<14); v < (1<<21); v++) { + for (uint64_t v = (1<<14) + start; v < (1<<21); v += stride) { unsigned char b[10]; size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); assert(out_s == 3); @@ -133,7 +135,7 @@ static void test_vlq_uint64(void) { } printf("%u\n", 1<<21); - for (uint64_t v = (1<<21); v < (1<<28); v++) { + for (uint64_t v = (1<<21) + start; v < (1<<28); v += stride) { unsigned char b[10]; size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); assert(out_s == 4); @@ -143,9 +145,11 @@ static void test_vlq_uint64(void) { } printf("%u\n", 1<<28); +#if USE_OPENMP #pragma omp parallel num_threads(4) #pragma omp for - for (uint64_t v = (1<<28); v < (1ULL<<35); v++) { +#endif + for (uint64_t v = (1<<28) + start; v < (1ULL<<35); v += stride) { unsigned char b[10]; size_t out_s = tokudb::vlq_encode_ui(v, b, sizeof b); assert(out_s == 5); @@ -155,7 +159,12 @@ static void test_vlq_uint64(void) { } } -int main(void) { - test_vlq_uint64(); +int main(int argc, char *argv[]) { + uint64_t start = 0, stride = 1; + if (argc == 3) { + start = atoll(argv[1]); + stride = atoll(argv[2]); + } + test_vlq_uint64(start, stride); return 0; } From ca67239c510ca15435b900ab708cd6e5e6ff286e Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Wed, 12 Nov 2014 14:36:08 -0500 Subject: [PATCH 21/36] speed up tokudb handler unit tests --- storage/tokudb/tests/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/tokudb/tests/Makefile b/storage/tokudb/tests/Makefile index 1943cc01867..721800e4840 100644 --- a/storage/tokudb/tests/Makefile +++ b/storage/tokudb/tests/Makefile @@ -14,6 +14,8 @@ VALGRIND = valgrind -q --leak-check=full --show-reachable=yes --suppressions=$(F ifeq ($(GCOV),1) CXXFLAGS += -fprofile-arcs -ftest-coverage +else +CXXFLAGS += -O3 endif all: $(TARGETS) From 8a7f07711e692e721fd787ca16e6401e3b9cd511 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Wed, 12 Nov 2014 18:30:16 -0500 Subject: [PATCH 22/36] increase test coverage of the cardinality code --- storage/tokudb/tests/Makefile | 4 +- storage/tokudb/tests/card_test.cc | 13 ++ storage/tokudb/tests/card_test_key_info.cc | 212 +++++++++++++++++++++ storage/tokudb/tests/fake_mysql.h | 2 +- 4 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 storage/tokudb/tests/card_test_key_info.cc diff --git a/storage/tokudb/tests/Makefile b/storage/tokudb/tests/Makefile index 721800e4840..47e2027e7b1 100644 --- a/storage/tokudb/tests/Makefile +++ b/storage/tokudb/tests/Makefile @@ -26,7 +26,7 @@ clean: %.check: % LD_LIBRARY_PATH=$(FRACTALTREE_INSTALL_DIR)/lib $(VALGRIND) ./$< -card.check: card_test.check card_1.check card_inf.check card_inf_1.check card_random_1.check card_etime.check +card.check: $(patsubst %.cc,%.check,$(wildcard card*.cc)) true ifndef USE_OPENMP @@ -37,7 +37,7 @@ vlq_test_uint64.check: $(foreach i,0 1 2 3 4 5 6 7,vlq_test_uint64_$(i).check) true endif -vlq.check: vlq_test.check vlq_test_uint32.check vlq_test_uint64.check +vlq.check: $(patsubst %.cc,%.check,$(wildcard vlq*.cc)) true max_test.check: max_test diff --git a/storage/tokudb/tests/card_test.cc b/storage/tokudb/tests/card_test.cc index fbf80bf7804..0bd78d78cc7 100644 --- a/storage/tokudb/tests/card_test.cc +++ b/storage/tokudb/tests/card_test.cc @@ -222,6 +222,19 @@ static void test_10(DB_ENV *env) { error = txn->commit(txn, 0); assert(error == 0); + + // test delete card + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + error = tokudb::delete_card_from_status(status_db, txn); + assert(error == 0); + + error = tokudb::get_card_from_status(status_db, txn, 0, NULL); + assert(error == DB_NOTFOUND); + + error = txn->commit(txn, 0); + assert(error == 0); error = tokudb::close_status(&status_db); assert(error == 0); diff --git a/storage/tokudb/tests/card_test_key_info.cc b/storage/tokudb/tests/card_test_key_info.cc new file mode 100644 index 00000000000..641f8a996db --- /dev/null +++ b/storage/tokudb/tests/card_test_key_info.cc @@ -0,0 +1,212 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." +#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it." +// test tokudb cardinality in status dictionary +#include +#include +#include +#include +#include +#include +#include +#include +typedef unsigned long long ulonglong; +#include +#include + +#include "fake_mysql.h" + +#if __APPLE__ +typedef unsigned long ulong; +#endif +#include +#define MAX_KEY 256 + +static void test_no_keys() { + TABLE_SHARE s = { 0, 0, 0, NULL }; + TABLE t = { &s, NULL }; + assert(tokudb::compute_total_key_parts(&s) == 0); + tokudb::set_card_in_key_info(&t, 0, NULL); +} + +static void test_simple_pk() { + const uint keys = 1; + const uint key_parts = 1; + uint64_t pk_rec_per_key[keys] = { 0 }; + KEY_INFO pk = { 0, key_parts, pk_rec_per_key, (char *) "PRIMARY" }; + TABLE_SHARE s = { 0, keys, key_parts, &pk }; + TABLE t = { &s, &pk }; + assert(tokudb::compute_total_key_parts(&s) == key_parts); + uint64_t computed_rec_per_key[keys] = { 2 }; + tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key); + assert(t.key_info[0].rec_per_key[0] == 1); +} + +static void test_pk_2() { + const uint keys = 1; + const uint key_parts = 2; + uint64_t pk_rec_per_key[keys * key_parts] = { 0 }; + KEY_INFO pk = { 0, key_parts, pk_rec_per_key, (char *) "PRIMARY" }; + TABLE_SHARE s = { 0, keys, key_parts, &pk }; + TABLE t = { &s, &pk }; + assert(tokudb::compute_total_key_parts(&s) == key_parts); + uint64_t computed_rec_per_key[keys * key_parts] = { 2, 3 }; + tokudb::set_card_in_key_info(&t, keys * key_parts, computed_rec_per_key); + assert(t.key_info[0].rec_per_key[0] == 2); + assert(t.key_info[0].rec_per_key[1] == 1); +} + +static void test_simple_sk() { + const uint keys = 1; + const uint key_parts = 1; + uint64_t sk_rec_per_key[keys] = { 0 }; + KEY_INFO sk = { 0, keys, sk_rec_per_key, (char *) "KEY" }; + TABLE_SHARE s = { MAX_KEY, keys, key_parts, &sk }; + TABLE t = { &s, &sk }; + assert(tokudb::compute_total_key_parts(&s) == 1); + uint64_t computed_rec_per_key[keys] = { 2 }; + tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key); + assert(t.key_info[0].rec_per_key[0] == 2); +} + +static void test_simple_unique_sk() { + const uint keys = 1; + uint64_t sk_rec_per_key[keys] = { 0 }; + KEY_INFO sk = { HA_NOSAME, keys, sk_rec_per_key, (char *) "KEY" }; + TABLE_SHARE s = { MAX_KEY, keys, keys, &sk }; + TABLE t = { &s, &sk }; + assert(tokudb::compute_total_key_parts(&s) == 1); + uint64_t computed_rec_per_key[keys] = { 2 }; + tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key); + assert(t.key_info[0].rec_per_key[0] == 1); +} + +static void test_simple_pk_sk() { + const uint keys = 2; + uint64_t rec_per_key[keys] = { 0 }; + KEY_INFO key_info[keys] = { + { 0, 1, &rec_per_key[0], (char *) "PRIMARY" }, + { 0, 1, &rec_per_key[1], (char *) "KEY" }, + }; + TABLE_SHARE s = { 0, keys, keys, key_info }; + TABLE t = { &s, key_info }; + assert(tokudb::compute_total_key_parts(&s) == 2); + uint64_t computed_rec_per_key[keys] = { 100, 200 }; + tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key); + assert(t.key_info[0].rec_per_key[0] == 1); + assert(t.key_info[1].rec_per_key[0] == 200); +} + +static void test_simple_sk_pk() { + const uint keys = 2; + uint64_t rec_per_key[keys] = { 0 }; + KEY_INFO key_info[keys] = { + { 0, 1, &rec_per_key[0], (char *) "KEY" }, + { 0, 1, &rec_per_key[1], (char *) "PRIMARY" }, + }; + TABLE_SHARE s = { 1, keys, keys, key_info }; + TABLE t = { &s, key_info }; + assert(tokudb::compute_total_key_parts(&s) == 2); + uint64_t computed_rec_per_key[keys] = { 100, 200 }; + tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key); + assert(t.key_info[0].rec_per_key[0] == 100); + assert(t.key_info[1].rec_per_key[0] == 1); +} + +int main() { + test_no_keys(); + test_simple_pk(); + test_pk_2(); + test_simple_sk(); + test_simple_unique_sk(); + test_simple_pk_sk(); + test_simple_sk_pk(); + return 0; +} diff --git a/storage/tokudb/tests/fake_mysql.h b/storage/tokudb/tests/fake_mysql.h index 46aa41099ff..39d4f59160f 100644 --- a/storage/tokudb/tests/fake_mysql.h +++ b/storage/tokudb/tests/fake_mysql.h @@ -110,5 +110,5 @@ public: }; uint get_key_parts(KEY_INFO *key_info) { assert(key_info); - return 0; + return key_info->key_parts; } From 2494bde757c78fe7a975e44c6ecb728e1e517216 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Wed, 12 Nov 2014 21:06:51 -0500 Subject: [PATCH 23/36] DB-759 fix tokudb::alter_card to copy ALL of the cardinality data not just the low byte --- storage/tokudb/tests/card_test_alter.cc | 566 +++++++++++++++++++++ storage/tokudb/tests/card_test_key_info.cc | 1 - storage/tokudb/tests/fake_mysql.h | 1 + 3 files changed, 567 insertions(+), 1 deletion(-) create mode 100644 storage/tokudb/tests/card_test_alter.cc diff --git a/storage/tokudb/tests/card_test_alter.cc b/storage/tokudb/tests/card_test_alter.cc new file mode 100644 index 00000000000..549996ba58b --- /dev/null +++ b/storage/tokudb/tests/card_test_alter.cc @@ -0,0 +1,566 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." +#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it." +// test tokudb cardinality in status dictionary +#include +#include +#include +#include +#include +#include +#include +#include +typedef unsigned long long ulonglong; +#include +#include + +#include "fake_mysql.h" + +#if __APPLE__ +typedef unsigned long ulong; +#endif +#include + +static void test_no_keys(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_no_keys", txn); + assert(error == 0); + + const uint keys = 0; + const uint key_parts = 0; + TABLE_SHARE s = { MAX_KEY, keys, key_parts, NULL }; + + error = tokudb::alter_card(status_db, txn, &s, &s); + assert(error == 0); + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + +static void test_keys(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_keys", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 3; + const uint ta_key_parts = 1; + const int ta_rec_per_keys = ta_keys * ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 1000, 2000, 3000, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, ta_key_parts, &ta_rec_per_key[0], (char *) "key_a" }, + { 0, ta_key_parts, &ta_rec_per_key[1], (char *) "key_b" }, + { 0, ta_key_parts, &ta_rec_per_key[1], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &ta); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[ta_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, ta_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < ta_rec_per_keys; i++) { + assert(current_rec_per_key[i] == ta_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + +static void test_drop_0(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_drop_0", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 3; + const uint ta_key_parts = 1; + const uint ta_rec_per_keys = ta_keys * ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 1000, 2000, 3000, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, ta_key_parts, &ta_rec_per_key[0], (char *) "key_a" }, + { 0, ta_key_parts, &ta_rec_per_key[1], (char *) "key_b" }, + { 0, ta_key_parts, &ta_rec_per_key[2], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + const uint tb_keys = 2; + const uint tb_key_parts = 1; + const int tb_rec_per_keys = tb_keys * tb_key_parts; + uint64_t tb_rec_per_key[tb_rec_per_keys] = { + 2000, 3000, + }; + KEY_INFO tb_key_info[tb_rec_per_keys] = { + { 0, tb_key_parts, &tb_rec_per_key[0], (char *) "key_b" }, + { 0, tb_key_parts, &tb_rec_per_key[1], (char *) "key_c" }, + }; + TABLE_SHARE tb = { MAX_KEY, tb_keys, tb_key_parts, tb_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &tb); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[tb_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, tb_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < tb_rec_per_keys; i++) { + assert(current_rec_per_key[i] == tb_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + +static void test_drop_1(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_drop_1", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 3; + const uint ta_key_parts = 1; + const uint ta_rec_per_keys = ta_keys * ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 1000, 2000, 3000, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, ta_key_parts, &ta_rec_per_key[0], (char *) "key_a" }, + { 0, ta_key_parts, &ta_rec_per_key[1], (char *) "key_b" }, + { 0, ta_key_parts, &ta_rec_per_key[2], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + const uint tb_keys = 2; + const uint tb_key_parts = 1; + const int tb_rec_per_keys = tb_keys * tb_key_parts; + uint64_t tb_rec_per_key[tb_rec_per_keys] = { + 1000, 3000, + }; + KEY_INFO tb_key_info[tb_rec_per_keys] = { + { 0, tb_key_parts, &tb_rec_per_key[0], (char *) "key_a" }, + { 0, tb_key_parts, &tb_rec_per_key[1], (char *) "key_c" }, + }; + TABLE_SHARE tb = { MAX_KEY, tb_keys, tb_key_parts, tb_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &tb); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[tb_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, tb_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < tb_rec_per_keys; i++) { + assert(current_rec_per_key[i] == tb_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + +static void test_drop_2(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_drop_2", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 3; + const uint ta_key_parts = 1; + const uint ta_rec_per_keys = ta_keys * ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 1000, 2000, 3000, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, ta_key_parts, &ta_rec_per_key[0], (char *) "key_a" }, + { 0, ta_key_parts, &ta_rec_per_key[1], (char *) "key_b" }, + { 0, ta_key_parts, &ta_rec_per_key[2], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + const uint tb_keys = 2; + const uint tb_key_parts = 1; + const int tb_rec_per_keys = tb_keys * tb_key_parts; + uint64_t tb_rec_per_key[tb_rec_per_keys] = { + 1000, 2000, + }; + KEY_INFO tb_key_info[tb_rec_per_keys] = { + { 0, tb_key_parts, &tb_rec_per_key[0], (char *) "key_a" }, + { 0, tb_key_parts, &tb_rec_per_key[1], (char *) "key_b" }, + }; + TABLE_SHARE tb = { MAX_KEY, tb_keys, tb_key_parts, tb_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &tb); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[tb_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, tb_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < tb_rec_per_keys; i++) { + assert(current_rec_per_key[i] == tb_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + +static void test_add_0(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_add_0", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 2; + const uint ta_key_parts = 1; + const uint ta_rec_per_keys = ta_keys * ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 2000, 3000, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, ta_key_parts, &ta_rec_per_key[0], (char *) "key_b" }, + { 0, ta_key_parts, &ta_rec_per_key[1], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + const uint tb_keys = 3; + const uint tb_key_parts = 1; + const int tb_rec_per_keys = tb_keys * tb_key_parts; + uint64_t tb_rec_per_key[tb_rec_per_keys] = { + 0 /*not computed*/, 2000, 3000, + }; + KEY_INFO tb_key_info[tb_rec_per_keys] = { + { 0, tb_key_parts, &tb_rec_per_key[0], (char *) "key_a" }, + { 0, tb_key_parts, &tb_rec_per_key[1], (char *) "key_b" }, + { 0, tb_key_parts, &tb_rec_per_key[2], (char *) "key_c" }, + }; + TABLE_SHARE tb = { MAX_KEY, tb_keys, tb_key_parts, tb_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &tb); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[tb_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, tb_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < tb_rec_per_keys; i++) { + assert(current_rec_per_key[i] == tb_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + +static void test_add_1(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_add_1", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 2; + const uint ta_key_parts = 1; + const uint ta_rec_per_keys = ta_keys * ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 2000, 3000, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, ta_key_parts, &ta_rec_per_key[0], (char *) "key_b" }, + { 0, ta_key_parts, &ta_rec_per_key[1], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + const uint tb_keys = 3; + const uint tb_key_parts = 1; + const int tb_rec_per_keys = tb_keys * tb_key_parts; + uint64_t tb_rec_per_key[tb_rec_per_keys] = { + 2000, 0 /*not computed*/, 3000, + }; + KEY_INFO tb_key_info[tb_rec_per_keys] = { + { 0, tb_key_parts, &tb_rec_per_key[0], (char *) "key_b" }, + { 0, tb_key_parts, &tb_rec_per_key[1], (char *) "key_a" }, + { 0, tb_key_parts, &tb_rec_per_key[2], (char *) "key_c" }, + }; + TABLE_SHARE tb = { MAX_KEY, tb_keys, tb_key_parts, tb_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &tb); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[tb_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, tb_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < tb_rec_per_keys; i++) { + assert(current_rec_per_key[i] == tb_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + +static void test_add_2(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_add_2", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 2; + const uint ta_key_parts = 1; + const uint ta_rec_per_keys = ta_keys * ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 2000, 3000, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, ta_key_parts, &ta_rec_per_key[0], (char *) "key_b" }, + { 0, ta_key_parts, &ta_rec_per_key[1], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + const uint tb_keys = 3; + const uint tb_key_parts = 1; + const int tb_rec_per_keys = tb_keys * tb_key_parts; + uint64_t tb_rec_per_key[tb_rec_per_keys] = { + 2000, 3000, 0 /*not computed*/, + }; + KEY_INFO tb_key_info[tb_rec_per_keys] = { + { 0, tb_key_parts, &tb_rec_per_key[0], (char *) "key_b" }, + { 0, tb_key_parts, &tb_rec_per_key[1], (char *) "key_c" }, + { 0, tb_key_parts, &tb_rec_per_key[2], (char *) "key_a" }, + }; + TABLE_SHARE tb = { MAX_KEY, tb_keys, tb_key_parts, tb_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &tb); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[tb_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, tb_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < tb_rec_per_keys; i++) { + assert(current_rec_per_key[i] == tb_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + +int main() { + int error; + + error = system("rm -rf " __FILE__ ".testdir"); + assert(error == 0); + + error = mkdir(__FILE__ ".testdir", S_IRWXU+S_IRWXG+S_IRWXO); + assert(error == 0); + + DB_ENV *env = NULL; + error = db_env_create(&env, 0); + assert(error == 0); + + error = env->open(env, __FILE__ ".testdir", DB_INIT_MPOOL + DB_INIT_LOG + DB_INIT_LOCK + DB_INIT_TXN + DB_PRIVATE + DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); + assert(error == 0); + + test_no_keys(env); + test_keys(env); + test_drop_0(env); + test_drop_1(env); + test_drop_2(env); + test_add_0(env); + test_add_1(env); + test_add_2(env); + + error = env->close(env, 0); + assert(error == 0); + + return 0; +} diff --git a/storage/tokudb/tests/card_test_key_info.cc b/storage/tokudb/tests/card_test_key_info.cc index 641f8a996db..68f61942397 100644 --- a/storage/tokudb/tests/card_test_key_info.cc +++ b/storage/tokudb/tests/card_test_key_info.cc @@ -107,7 +107,6 @@ typedef unsigned long long ulonglong; typedef unsigned long ulong; #endif #include -#define MAX_KEY 256 static void test_no_keys() { TABLE_SHARE s = { 0, 0, 0, NULL }; diff --git a/storage/tokudb/tests/fake_mysql.h b/storage/tokudb/tests/fake_mysql.h index 39d4f59160f..82be0f01de0 100644 --- a/storage/tokudb/tests/fake_mysql.h +++ b/storage/tokudb/tests/fake_mysql.h @@ -112,3 +112,4 @@ uint get_key_parts(KEY_INFO *key_info) { assert(key_info); return key_info->key_parts; } +#define MAX_KEY (1U << 30) From 50c5339272403dfac435c7df632274ad8b18aaf8 Mon Sep 17 00:00:00 2001 From: Sergey Petrunya Date: Thu, 13 Nov 2014 14:12:41 +0300 Subject: [PATCH 24/36] MDEV-7068: MRR accessing uninitialised bytes, test case failure main.innodb_mrr Backport to 5.3: - Don't call index_reader->interrupt_read() if the index reader has returned all rows that matched its keys. --- sql/multi_range_read.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc index 6dfd8bec980..eb1de09e056 100644 --- a/sql/multi_range_read.cc +++ b/sql/multi_range_read.cc @@ -668,8 +668,19 @@ int Mrr_ordered_rndpos_reader::refill_from_index_reader() rowid_buffer->write_ptr2= (uchar*)&range_info; rowid_buffer->write(); } - - index_reader->interrupt_read(); + + /* + When index_reader_needs_refill=TRUE, this means we've got all of index + tuples for lookups keys that index_reader had. We are not in the middle + of an index read, so there is no need to call interrupt_read. + + Actually, we must not call interrupt_read(), because it could be that we + haven't read a single row (because all index lookups returned + HA_ERR_KEY_NOT_FOUND). In this case, interrupt_read() will cause [harmless] + valgrind warnings when trying to save garbage from table->record[0]. + */ + if (!index_reader_needs_refill) + index_reader->interrupt_read(); /* Sort the buffer contents by rowid */ rowid_buffer->sort((qsort2_cmp)rowid_cmp_reverse, (void*)file); From 50b928b046edbb4355d73b5bdbc1bf12479872d4 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Thu, 13 Nov 2014 10:53:22 -0500 Subject: [PATCH 25/36] DB-759 test and fix alter table bug with cardinality data --- .../suite/tokudb/r/card_drop_index_2.result | 158 ++++++++++++++++++ .../suite/tokudb/t/card_drop_index_2.test | 35 ++++ storage/tokudb/ha_tokudb_alter_56.cc | 2 +- storage/tokudb/tests/card_test_alter.cc | 142 ++++++++++++++-- storage/tokudb/tokudb_card.h | 9 +- 5 files changed, 332 insertions(+), 14 deletions(-) create mode 100644 mysql-test/suite/tokudb/r/card_drop_index_2.result create mode 100644 mysql-test/suite/tokudb/t/card_drop_index_2.test diff --git a/mysql-test/suite/tokudb/r/card_drop_index_2.result b/mysql-test/suite/tokudb/r/card_drop_index_2.result new file mode 100644 index 00000000000..4103a37a6ed --- /dev/null +++ b/mysql-test/suite/tokudb/r/card_drop_index_2.result @@ -0,0 +1,158 @@ +set default_storage_engine='tokudb'; +drop table if exists tt; +create table tt (a int, b int, c int, primary key(a), key(b), key(c)); +insert into tt values (0, 0, 0), (0+1, 0, 0), (0+2, 0, 0), (0+3, 0, 0); +insert into tt values (4, 4, 0), (4+1, 4, 0), (4+2, 4, 0), (4+3, 4, 0); +insert into tt values (8, 8, 0), (8+1, 8, 0), (8+2, 8, 0), (8+3, 8, 0); +insert into tt values (12, 12, 0), (12+1, 12, 0), (12+2, 12, 0), (12+3, 12, 0); +insert into tt values (16, 16, 0), (16+1, 16, 0), (16+2, 16, 0), (16+3, 16, 0); +insert into tt values (20, 20, 0), (20+1, 20, 0), (20+2, 20, 0), (20+3, 20, 0); +insert into tt values (24, 24, 0), (24+1, 24, 0), (24+2, 24, 0), (24+3, 24, 0); +insert into tt values (28, 28, 0), (28+1, 28, 0), (28+2, 28, 0), (28+3, 28, 0); +insert into tt values (32, 32, 0), (32+1, 32, 0), (32+2, 32, 0), (32+3, 32, 0); +insert into tt values (36, 36, 0), (36+1, 36, 0), (36+2, 36, 0), (36+3, 36, 0); +insert into tt values (40, 40, 0), (40+1, 40, 0), (40+2, 40, 0), (40+3, 40, 0); +insert into tt values (44, 44, 0), (44+1, 44, 0), (44+2, 44, 0), (44+3, 44, 0); +insert into tt values (48, 48, 0), (48+1, 48, 0), (48+2, 48, 0), (48+3, 48, 0); +insert into tt values (52, 52, 0), (52+1, 52, 0), (52+2, 52, 0), (52+3, 52, 0); +insert into tt values (56, 56, 0), (56+1, 56, 0), (56+2, 56, 0), (56+3, 56, 0); +insert into tt values (60, 60, 0), (60+1, 60, 0), (60+2, 60, 0), (60+3, 60, 0); +insert into tt values (64, 64, 0), (64+1, 64, 0), (64+2, 64, 0), (64+3, 64, 0); +insert into tt values (68, 68, 0), (68+1, 68, 0), (68+2, 68, 0), (68+3, 68, 0); +insert into tt values (72, 72, 0), (72+1, 72, 0), (72+2, 72, 0), (72+3, 72, 0); +insert into tt values (76, 76, 0), (76+1, 76, 0), (76+2, 76, 0), (76+3, 76, 0); +insert into tt values (80, 80, 0), (80+1, 80, 0), (80+2, 80, 0), (80+3, 80, 0); +insert into tt values (84, 84, 0), (84+1, 84, 0), (84+2, 84, 0), (84+3, 84, 0); +insert into tt values (88, 88, 0), (88+1, 88, 0), (88+2, 88, 0), (88+3, 88, 0); +insert into tt values (92, 92, 0), (92+1, 92, 0), (92+2, 92, 0), (92+3, 92, 0); +insert into tt values (96, 96, 0), (96+1, 96, 0), (96+2, 96, 0), (96+3, 96, 0); +insert into tt values (100, 100, 0), (100+1, 100, 0), (100+2, 100, 0), (100+3, 100, 0); +insert into tt values (104, 104, 0), (104+1, 104, 0), (104+2, 104, 0), (104+3, 104, 0); +insert into tt values (108, 108, 0), (108+1, 108, 0), (108+2, 108, 0), (108+3, 108, 0); +insert into tt values (112, 112, 0), (112+1, 112, 0), (112+2, 112, 0), (112+3, 112, 0); +insert into tt values (116, 116, 0), (116+1, 116, 0), (116+2, 116, 0), (116+3, 116, 0); +insert into tt values (120, 120, 0), (120+1, 120, 0), (120+2, 120, 0), (120+3, 120, 0); +insert into tt values (124, 124, 0), (124+1, 124, 0), (124+2, 124, 0), (124+3, 124, 0); +insert into tt values (128, 128, 0), (128+1, 128, 0), (128+2, 128, 0), (128+3, 128, 0); +insert into tt values (132, 132, 0), (132+1, 132, 0), (132+2, 132, 0), (132+3, 132, 0); +insert into tt values (136, 136, 0), (136+1, 136, 0), (136+2, 136, 0), (136+3, 136, 0); +insert into tt values (140, 140, 0), (140+1, 140, 0), (140+2, 140, 0), (140+3, 140, 0); +insert into tt values (144, 144, 0), (144+1, 144, 0), (144+2, 144, 0), (144+3, 144, 0); +insert into tt values (148, 148, 0), (148+1, 148, 0), (148+2, 148, 0), (148+3, 148, 0); +insert into tt values (152, 152, 0), (152+1, 152, 0), (152+2, 152, 0), (152+3, 152, 0); +insert into tt values (156, 156, 0), (156+1, 156, 0), (156+2, 156, 0), (156+3, 156, 0); +insert into tt values (160, 160, 0), (160+1, 160, 0), (160+2, 160, 0), (160+3, 160, 0); +insert into tt values (164, 164, 0), (164+1, 164, 0), (164+2, 164, 0), (164+3, 164, 0); +insert into tt values (168, 168, 0), (168+1, 168, 0), (168+2, 168, 0), (168+3, 168, 0); +insert into tt values (172, 172, 0), (172+1, 172, 0), (172+2, 172, 0), (172+3, 172, 0); +insert into tt values (176, 176, 0), (176+1, 176, 0), (176+2, 176, 0), (176+3, 176, 0); +insert into tt values (180, 180, 0), (180+1, 180, 0), (180+2, 180, 0), (180+3, 180, 0); +insert into tt values (184, 184, 0), (184+1, 184, 0), (184+2, 184, 0), (184+3, 184, 0); +insert into tt values (188, 188, 0), (188+1, 188, 0), (188+2, 188, 0), (188+3, 188, 0); +insert into tt values (192, 192, 0), (192+1, 192, 0), (192+2, 192, 0), (192+3, 192, 0); +insert into tt values (196, 196, 0), (196+1, 196, 0), (196+2, 196, 0), (196+3, 196, 0); +insert into tt values (200, 200, 0), (200+1, 200, 0), (200+2, 200, 0), (200+3, 200, 0); +insert into tt values (204, 204, 0), (204+1, 204, 0), (204+2, 204, 0), (204+3, 204, 0); +insert into tt values (208, 208, 0), (208+1, 208, 0), (208+2, 208, 0), (208+3, 208, 0); +insert into tt values (212, 212, 0), (212+1, 212, 0), (212+2, 212, 0), (212+3, 212, 0); +insert into tt values (216, 216, 0), (216+1, 216, 0), (216+2, 216, 0), (216+3, 216, 0); +insert into tt values (220, 220, 0), (220+1, 220, 0), (220+2, 220, 0), (220+3, 220, 0); +insert into tt values (224, 224, 0), (224+1, 224, 0), (224+2, 224, 0), (224+3, 224, 0); +insert into tt values (228, 228, 0), (228+1, 228, 0), (228+2, 228, 0), (228+3, 228, 0); +insert into tt values (232, 232, 0), (232+1, 232, 0), (232+2, 232, 0), (232+3, 232, 0); +insert into tt values (236, 236, 0), (236+1, 236, 0), (236+2, 236, 0), (236+3, 236, 0); +insert into tt values (240, 240, 0), (240+1, 240, 0), (240+2, 240, 0), (240+3, 240, 0); +insert into tt values (244, 244, 0), (244+1, 244, 0), (244+2, 244, 0), (244+3, 244, 0); +insert into tt values (248, 248, 0), (248+1, 248, 0), (248+2, 248, 0), (248+3, 248, 0); +insert into tt values (252, 252, 0), (252+1, 252, 0), (252+2, 252, 0), (252+3, 252, 0); +insert into tt values (256, 256, 0), (256+1, 256, 0), (256+2, 256, 0), (256+3, 256, 0); +insert into tt values (260, 260, 0), (260+1, 260, 0), (260+2, 260, 0), (260+3, 260, 0); +insert into tt values (264, 264, 0), (264+1, 264, 0), (264+2, 264, 0), (264+3, 264, 0); +insert into tt values (268, 268, 0), (268+1, 268, 0), (268+2, 268, 0), (268+3, 268, 0); +insert into tt values (272, 272, 0), (272+1, 272, 0), (272+2, 272, 0), (272+3, 272, 0); +insert into tt values (276, 276, 0), (276+1, 276, 0), (276+2, 276, 0), (276+3, 276, 0); +insert into tt values (280, 280, 0), (280+1, 280, 0), (280+2, 280, 0), (280+3, 280, 0); +insert into tt values (284, 284, 0), (284+1, 284, 0), (284+2, 284, 0), (284+3, 284, 0); +insert into tt values (288, 288, 0), (288+1, 288, 0), (288+2, 288, 0), (288+3, 288, 0); +insert into tt values (292, 292, 0), (292+1, 292, 0), (292+2, 292, 0), (292+3, 292, 0); +insert into tt values (296, 296, 0), (296+1, 296, 0), (296+2, 296, 0), (296+3, 296, 0); +insert into tt values (300, 300, 0), (300+1, 300, 0), (300+2, 300, 0), (300+3, 300, 0); +insert into tt values (304, 304, 0), (304+1, 304, 0), (304+2, 304, 0), (304+3, 304, 0); +insert into tt values (308, 308, 0), (308+1, 308, 0), (308+2, 308, 0), (308+3, 308, 0); +insert into tt values (312, 312, 0), (312+1, 312, 0), (312+2, 312, 0), (312+3, 312, 0); +insert into tt values (316, 316, 0), (316+1, 316, 0), (316+2, 316, 0), (316+3, 316, 0); +insert into tt values (320, 320, 0), (320+1, 320, 0), (320+2, 320, 0), (320+3, 320, 0); +insert into tt values (324, 324, 0), (324+1, 324, 0), (324+2, 324, 0), (324+3, 324, 0); +insert into tt values (328, 328, 0), (328+1, 328, 0), (328+2, 328, 0), (328+3, 328, 0); +insert into tt values (332, 332, 0), (332+1, 332, 0), (332+2, 332, 0), (332+3, 332, 0); +insert into tt values (336, 336, 0), (336+1, 336, 0), (336+2, 336, 0), (336+3, 336, 0); +insert into tt values (340, 340, 0), (340+1, 340, 0), (340+2, 340, 0), (340+3, 340, 0); +insert into tt values (344, 344, 0), (344+1, 344, 0), (344+2, 344, 0), (344+3, 344, 0); +insert into tt values (348, 348, 0), (348+1, 348, 0), (348+2, 348, 0), (348+3, 348, 0); +insert into tt values (352, 352, 0), (352+1, 352, 0), (352+2, 352, 0), (352+3, 352, 0); +insert into tt values (356, 356, 0), (356+1, 356, 0), (356+2, 356, 0), (356+3, 356, 0); +insert into tt values (360, 360, 0), (360+1, 360, 0), (360+2, 360, 0), (360+3, 360, 0); +insert into tt values (364, 364, 0), (364+1, 364, 0), (364+2, 364, 0), (364+3, 364, 0); +insert into tt values (368, 368, 0), (368+1, 368, 0), (368+2, 368, 0), (368+3, 368, 0); +insert into tt values (372, 372, 0), (372+1, 372, 0), (372+2, 372, 0), (372+3, 372, 0); +insert into tt values (376, 376, 0), (376+1, 376, 0), (376+2, 376, 0), (376+3, 376, 0); +insert into tt values (380, 380, 0), (380+1, 380, 0), (380+2, 380, 0), (380+3, 380, 0); +insert into tt values (384, 384, 0), (384+1, 384, 0), (384+2, 384, 0), (384+3, 384, 0); +insert into tt values (388, 388, 0), (388+1, 388, 0), (388+2, 388, 0), (388+3, 388, 0); +insert into tt values (392, 392, 0), (392+1, 392, 0), (392+2, 392, 0), (392+3, 392, 0); +insert into tt values (396, 396, 0), (396+1, 396, 0), (396+2, 396, 0), (396+3, 396, 0); +insert into tt values (400, 400, 0), (400+1, 400, 0), (400+2, 400, 0), (400+3, 400, 0); +insert into tt values (404, 404, 0), (404+1, 404, 0), (404+2, 404, 0), (404+3, 404, 0); +insert into tt values (408, 408, 0), (408+1, 408, 0), (408+2, 408, 0), (408+3, 408, 0); +insert into tt values (412, 412, 0), (412+1, 412, 0), (412+2, 412, 0), (412+3, 412, 0); +insert into tt values (416, 416, 0), (416+1, 416, 0), (416+2, 416, 0), (416+3, 416, 0); +insert into tt values (420, 420, 0), (420+1, 420, 0), (420+2, 420, 0), (420+3, 420, 0); +insert into tt values (424, 424, 0), (424+1, 424, 0), (424+2, 424, 0), (424+3, 424, 0); +insert into tt values (428, 428, 0), (428+1, 428, 0), (428+2, 428, 0), (428+3, 428, 0); +insert into tt values (432, 432, 0), (432+1, 432, 0), (432+2, 432, 0), (432+3, 432, 0); +insert into tt values (436, 436, 0), (436+1, 436, 0), (436+2, 436, 0), (436+3, 436, 0); +insert into tt values (440, 440, 0), (440+1, 440, 0), (440+2, 440, 0), (440+3, 440, 0); +insert into tt values (444, 444, 0), (444+1, 444, 0), (444+2, 444, 0), (444+3, 444, 0); +insert into tt values (448, 448, 0), (448+1, 448, 0), (448+2, 448, 0), (448+3, 448, 0); +insert into tt values (452, 452, 0), (452+1, 452, 0), (452+2, 452, 0), (452+3, 452, 0); +insert into tt values (456, 456, 0), (456+1, 456, 0), (456+2, 456, 0), (456+3, 456, 0); +insert into tt values (460, 460, 0), (460+1, 460, 0), (460+2, 460, 0), (460+3, 460, 0); +insert into tt values (464, 464, 0), (464+1, 464, 0), (464+2, 464, 0), (464+3, 464, 0); +insert into tt values (468, 468, 0), (468+1, 468, 0), (468+2, 468, 0), (468+3, 468, 0); +insert into tt values (472, 472, 0), (472+1, 472, 0), (472+2, 472, 0), (472+3, 472, 0); +insert into tt values (476, 476, 0), (476+1, 476, 0), (476+2, 476, 0), (476+3, 476, 0); +insert into tt values (480, 480, 0), (480+1, 480, 0), (480+2, 480, 0), (480+3, 480, 0); +insert into tt values (484, 484, 0), (484+1, 484, 0), (484+2, 484, 0), (484+3, 484, 0); +insert into tt values (488, 488, 0), (488+1, 488, 0), (488+2, 488, 0), (488+3, 488, 0); +insert into tt values (492, 492, 0), (492+1, 492, 0), (492+2, 492, 0), (492+3, 492, 0); +insert into tt values (496, 496, 0), (496+1, 496, 0), (496+2, 496, 0), (496+3, 496, 0); +select count(*) from tt; +count(*) +500 +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 500 NULL NULL BTREE +tt 1 b 1 b A NULL NULL NULL YES BTREE +tt 1 c 1 c A NULL NULL NULL YES BTREE +analyze table tt; +Table Op Msg_type Msg_text +test.tt analyze status OK +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 500 NULL NULL BTREE +tt 1 b 1 b A 125 NULL NULL YES BTREE +tt 1 c 1 c A 1 NULL NULL YES BTREE +alter table tt drop key b; +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 500 NULL NULL BTREE +tt 1 c 1 c A 1 NULL NULL YES BTREE +alter table tt drop key c; +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 500 NULL NULL BTREE +flush tables; +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 500 NULL NULL BTREE +drop table tt; diff --git a/mysql-test/suite/tokudb/t/card_drop_index_2.test b/mysql-test/suite/tokudb/t/card_drop_index_2.test new file mode 100644 index 00000000000..88e0f6a413f --- /dev/null +++ b/mysql-test/suite/tokudb/t/card_drop_index_2.test @@ -0,0 +1,35 @@ +# test that add index keeps cardinality for older indexes + +source include/have_tokudb.inc; +set default_storage_engine='tokudb'; + +disable_warnings; +drop table if exists tt; +enable_warnings; + +create table tt (a int, b int, c int, primary key(a), key(b), key(c)); +let $a=0; +while ($a < 500) { + eval insert into tt values ($a, $a, 0), ($a+1, $a, 0), ($a+2, $a, 0), ($a+3, $a, 0); + inc $a; inc $a; inc $a; inc $a; +} +select count(*) from tt; + +# compute cardinality +show indexes from tt; +analyze table tt; +show indexes from tt; + +# drop key b +alter table tt drop key b; +show indexes from tt; + +# drop key c +alter table tt drop key c; +show indexes from tt; + +# test that cardinality is persistent +flush tables; +show indexes from tt; + +drop table tt; diff --git a/storage/tokudb/ha_tokudb_alter_56.cc b/storage/tokudb/ha_tokudb_alter_56.cc index e55b7b48ad8..1a03dc815a1 100644 --- a/storage/tokudb/ha_tokudb_alter_56.cc +++ b/storage/tokudb/ha_tokudb_alter_56.cc @@ -528,7 +528,7 @@ bool ha_tokudb::inplace_alter_table(TABLE *altered_table, Alter_inplace_info *ha error = alter_table_expand_blobs(altered_table, ha_alter_info); if (error == 0 && ctx->reset_card) { - error = tokudb::set_card_from_status(share->status_block, ctx->alter_txn, table->s, altered_table->s); + error = tokudb::alter_card(share->status_block, ctx->alter_txn, table->s, altered_table->s); } if (error == 0 && ctx->optimize_needed) { error = do_optimize(ha_thd()); diff --git a/storage/tokudb/tests/card_test_alter.cc b/storage/tokudb/tests/card_test_alter.cc index 549996ba58b..2a3e9396b3f 100644 --- a/storage/tokudb/tests/card_test_alter.cc +++ b/storage/tokudb/tests/card_test_alter.cc @@ -118,7 +118,7 @@ static void test_no_keys(DB_ENV *env) { DB *status_db = NULL; error = tokudb::create_status(env, &status_db, "status_no_keys", txn); assert(error == 0); - + const uint keys = 0; const uint key_parts = 0; TABLE_SHARE s = { MAX_KEY, keys, key_parts, NULL }; @@ -143,7 +143,7 @@ static void test_keys(DB_ENV *env) { DB *status_db = NULL; error = tokudb::create_status(env, &status_db, "status_keys", txn); assert(error == 0); - + // define tables const uint ta_keys = 3; const uint ta_key_parts = 1; @@ -190,7 +190,7 @@ static void test_drop_0(DB_ENV *env) { DB *status_db = NULL; error = tokudb::create_status(env, &status_db, "status_drop_0", txn); assert(error == 0); - + // define tables const uint ta_keys = 3; const uint ta_key_parts = 1; @@ -249,7 +249,7 @@ static void test_drop_1(DB_ENV *env) { DB *status_db = NULL; error = tokudb::create_status(env, &status_db, "status_drop_1", txn); assert(error == 0); - + // define tables const uint ta_keys = 3; const uint ta_key_parts = 1; @@ -308,7 +308,7 @@ static void test_drop_2(DB_ENV *env) { DB *status_db = NULL; error = tokudb::create_status(env, &status_db, "status_drop_2", txn); assert(error == 0); - + // define tables const uint ta_keys = 3; const uint ta_key_parts = 1; @@ -357,6 +357,65 @@ static void test_drop_2(DB_ENV *env) { assert(error == 0); } +static void test_drop_1_multiple_parts(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_drop_1_multiple_parts", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 3; + const uint ta_key_parts = 1+2+3; + const uint ta_rec_per_keys = ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 1000, 2000, 2001, 3000, 3001, 3002, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, 1, &ta_rec_per_key[0], (char *) "key_a" }, + { 0, 2, &ta_rec_per_key[0+1], (char *) "key_b" }, + { 0, 3, &ta_rec_per_key[0+1+2], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + const uint tb_keys = 2; + const uint tb_key_parts = 1+3; + const int tb_rec_per_keys = tb_key_parts; + uint64_t tb_rec_per_key[tb_rec_per_keys] = { + 1000, 3000, 3001, 3002, + }; + KEY_INFO tb_key_info[tb_rec_per_keys] = { + { 0, 1, &tb_rec_per_key[0], (char *) "key_a" }, + { 0, 3, &tb_rec_per_key[0+1], (char *) "key_c" }, + }; + TABLE_SHARE tb = { MAX_KEY, tb_keys, tb_key_parts, tb_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &tb); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[tb_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, tb_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < tb_rec_per_keys; i++) { + assert(current_rec_per_key[i] == tb_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + static void test_add_0(DB_ENV *env) { int error; @@ -367,7 +426,7 @@ static void test_add_0(DB_ENV *env) { DB *status_db = NULL; error = tokudb::create_status(env, &status_db, "status_add_0", txn); assert(error == 0); - + // define tables const uint ta_keys = 2; const uint ta_key_parts = 1; @@ -426,7 +485,7 @@ static void test_add_1(DB_ENV *env) { DB *status_db = NULL; error = tokudb::create_status(env, &status_db, "status_add_1", txn); assert(error == 0); - + // define tables const uint ta_keys = 2; const uint ta_key_parts = 1; @@ -485,7 +544,7 @@ static void test_add_2(DB_ENV *env) { DB *status_db = NULL; error = tokudb::create_status(env, &status_db, "status_add_2", txn); assert(error == 0); - + // define tables const uint ta_keys = 2; const uint ta_key_parts = 1; @@ -503,7 +562,7 @@ static void test_add_2(DB_ENV *env) { const uint tb_key_parts = 1; const int tb_rec_per_keys = tb_keys * tb_key_parts; uint64_t tb_rec_per_key[tb_rec_per_keys] = { - 2000, 3000, 0 /*not computed*/, + 2000, 3000, 0 /*not computed*/, }; KEY_INFO tb_key_info[tb_rec_per_keys] = { { 0, tb_key_parts, &tb_rec_per_key[0], (char *) "key_b" }, @@ -534,6 +593,65 @@ static void test_add_2(DB_ENV *env) { assert(error == 0); } +static void test_add_0_multiple_parts(DB_ENV *env) { + int error; + + DB_TXN *txn = NULL; + error = env->txn_begin(env, NULL, &txn, 0); + assert(error == 0); + + DB *status_db = NULL; + error = tokudb::create_status(env, &status_db, "status_add_0_multiple_parts", txn); + assert(error == 0); + + // define tables + const uint ta_keys = 2; + const uint ta_key_parts = 3+4; + const uint ta_rec_per_keys = ta_key_parts; + uint64_t ta_rec_per_key[ta_rec_per_keys] = { + 2000, 2001, 2002, 3000, 3001, 3002, 3003, + }; + KEY_INFO ta_key_info[ta_rec_per_keys] = { + { 0, 3, &ta_rec_per_key[0], (char *) "key_b" }, + { 0, 4, &ta_rec_per_key[3], (char *) "key_c" }, + }; + TABLE_SHARE ta = { MAX_KEY, ta_keys, ta_key_parts, ta_key_info }; + + const uint tb_keys = 3; + const uint tb_key_parts = 2+3+4; + const int tb_rec_per_keys = tb_key_parts; + uint64_t tb_rec_per_key[tb_rec_per_keys] = { + 0, 0 /*not computed*/, 2000, 2001, 2002, 3000, 3001, 3002, 3003, + }; + KEY_INFO tb_key_info[tb_rec_per_keys] = { + { 0, 2, &tb_rec_per_key[0], (char *) "key_a" }, + { 0, 3, &tb_rec_per_key[0+2], (char *) "key_b" }, + { 0, 4, &tb_rec_per_key[0+2+3], (char *) "key_c" }, + }; + TABLE_SHARE tb = { MAX_KEY, tb_keys, tb_key_parts, tb_key_info }; + + // set initial cardinality + error = tokudb::set_card_in_status(status_db, txn, ta_rec_per_keys, ta_rec_per_key); + assert(error == 0); + + error = tokudb::alter_card(status_db, txn, &ta, &tb); + assert(error == 0); + + // verify + uint64_t current_rec_per_key[tb_rec_per_keys]; + error = tokudb::get_card_from_status(status_db, txn, tb_rec_per_keys, current_rec_per_key); + assert(error == 0); + for (uint i = 0; i < tb_rec_per_keys; i++) { + assert(current_rec_per_key[i] == tb_rec_per_key[i]); + } + + error = txn->commit(txn, 0); + assert(error == 0); + + error = tokudb::close_status(&status_db); + assert(error == 0); +} + int main() { int error; @@ -547,7 +665,7 @@ int main() { error = db_env_create(&env, 0); assert(error == 0); - error = env->open(env, __FILE__ ".testdir", DB_INIT_MPOOL + DB_INIT_LOG + DB_INIT_LOCK + DB_INIT_TXN + DB_PRIVATE + DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); + error = env->open(env, __FILE__ ".testdir", DB_INIT_MPOOL + DB_INIT_LOG + DB_INIT_LOCK + DB_INIT_TXN + DB_PRIVATE + DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert(error == 0); test_no_keys(env); @@ -555,12 +673,14 @@ int main() { test_drop_0(env); test_drop_1(env); test_drop_2(env); + test_drop_1_multiple_parts(env); test_add_0(env); test_add_1(env); test_add_2(env); + test_add_0_multiple_parts(env); error = env->close(env, 0); assert(error == 0); - + return 0; } diff --git a/storage/tokudb/tokudb_card.h b/storage/tokudb/tokudb_card.h index a9439c124eb..797c705bbaf 100644 --- a/storage/tokudb/tokudb_card.h +++ b/storage/tokudb/tokudb_card.h @@ -174,9 +174,14 @@ namespace tokudb { return false; } + static void copy_card(uint64_t *dest, uint64_t *src, size_t n) { + for (size_t i = 0; i < n; i++) + dest[i] = src[i]; + } + // Altered table cardinality = select cardinality data from current table cardinality for keys that exist // in the altered table and the current table. - int set_card_from_status(DB *status_db, DB_TXN *txn, TABLE_SHARE *table_share, TABLE_SHARE *altered_table_share) { + int alter_card(DB *status_db, DB_TXN *txn, TABLE_SHARE *table_share, TABLE_SHARE *altered_table_share) { int error; // read existing cardinality data from status uint table_total_key_parts = tokudb::compute_total_key_parts(table_share); @@ -201,7 +206,7 @@ namespace tokudb { uint ith_key_parts = get_key_parts(&altered_table_share->key_info[i]); uint orig_key_index; if (find_index_of_key(altered_table_share->key_info[i].name, table_share, &orig_key_index)) { - memcpy(&altered_rec_per_key[next_key_parts], &rec_per_key[orig_key_offset[orig_key_index]], ith_key_parts); + copy_card(&altered_rec_per_key[next_key_parts], &rec_per_key[orig_key_offset[orig_key_index]], ith_key_parts); } next_key_parts += ith_key_parts; } From 3fbe68e6f85e5876b57133eaf5db2bfcd2ef04f6 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Fri, 14 Nov 2014 13:10:07 -0500 Subject: [PATCH 26/36] DB-754 build with bundled jemalloc --- scripts/make.mysql.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/make.mysql.bash b/scripts/make.mysql.bash index 19ae233af99..5654c1c7926 100755 --- a/scripts/make.mysql.bash +++ b/scripts/make.mysql.bash @@ -156,6 +156,7 @@ function generate_cmake_cmd () { -D CMAKE_BUILD_TYPE=$cmake_build_type \ -D CMAKE_TOKUDB_REVISION=$ft_revision \ -D TOKUDB_VERSION=tokudb-${tokudb_version} \ + -D WITH_JEMALLOC=bundled \ -D BUILD_TESTING=OFF \ -D USE_GTAGS=OFF \ -D USE_CTAGS=OFF \ From c9742ceac5bd682e24a435c36524305eecca7950 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sat, 15 Nov 2014 21:30:16 +0400 Subject: [PATCH 27/36] MDEV-6883 ST_WITHIN crashes server if (0,0) is matched to POLYGON((0 0)). Fixed the case when a polygon contains a single-point ring. --- mysql-test/r/gis.result | 6 ++++++ mysql-test/t/gis.test | 6 ++++++ sql/spatial.cc | 8 +++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/gis.result b/mysql-test/r/gis.result index 6c4d117042a..1d4a1eea8ff 100644 --- a/mysql-test/r/gis.result +++ b/mysql-test/r/gis.result @@ -1540,3 +1540,9 @@ Warnings: Warning 1300 Invalid utf8 character string: 'E043' Warning 1300 Invalid utf8 character string: 'E043' drop table t1; +# +# MDEV-6883 ST_WITHIN crashes server if (0,0) is matched to POLYGON((0 0)) +# +select st_within(GeomFromText('Polygon((0 0))'), Point(0,0)); +st_within(GeomFromText('Polygon((0 0))'), Point(0,0)) +1 diff --git a/mysql-test/t/gis.test b/mysql-test/t/gis.test index 9e743a65cdb..34a0309b908 100644 --- a/mysql-test/t/gis.test +++ b/mysql-test/t/gis.test @@ -1398,3 +1398,9 @@ insert into t1 values(geomfromtext("POINT(0 9.2233720368548e18)")); insert into t1 values(geomfromtext("POINT(0 9.2233720368548e18)")); select equals(`a`,convert(`a` using utf8)) from `t1`; drop table t1; + + +--echo # +--echo # MDEV-6883 ST_WITHIN crashes server if (0,0) is matched to POLYGON((0 0)) +--echo # +select st_within(GeomFromText('Polygon((0 0))'), Point(0,0)); diff --git a/sql/spatial.cc b/sql/spatial.cc index 551c79d4d90..2c838379505 100644 --- a/sql/spatial.cc +++ b/sql/spatial.cc @@ -1233,11 +1233,15 @@ int Gis_polygon::store_shapes(Gcalc_shape_transporter *trn) const trn->start_ring(); get_point(&first_x, &first_y, data); data+= POINT_DATA_SIZE; - n_points--; + prev_x= first_x; prev_y= first_y; if (trn->add_point(first_x, first_y)) return 1; + + if (--n_points == 0) + goto single_point_ring; + while (--n_points) { double x, y; @@ -1262,6 +1266,8 @@ int Gis_polygon::store_shapes(Gcalc_shape_transporter *trn) const return 1; } data+= POINT_DATA_SIZE; + +single_point_ring: trn->complete_ring(); } From 241294b53a0ee815a1c349820b8c9a69f36f4164 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 18 Nov 2014 15:42:25 +0100 Subject: [PATCH 28/36] MDEV-6854 Typo in cmake/plugin.cmake --- cmake/plugin.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index 83f819836f8..bfc9a76f1da 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -65,7 +65,7 @@ MACRO(MYSQL_ADD_PLUGIN) ENDIF() IF(WITH_${plugin}_STORAGE_ENGINE - OR WITH_{$plugin} + OR WITH_${plugin} OR WITH_ALL OR WITH_MAX AND NOT WITHOUT_${plugin}_STORAGE_ENGINE From 84f25c25f260373b54941d9239e8b0d758990601 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 18 Nov 2014 15:42:32 +0100 Subject: [PATCH 29/36] MDEV-3940 Server crash or assertion `item->type() == Item::STRING_ITEM' failure on LOAD DATA through a view with statement binary logging A "field" could be either an Item_field or (if loading into a view) an Item_direct_ref that references Item_field. Also: when iterating fields, use fields of the TABLE_LIST (table or view), not fields of a TABLE (actual underlying table - might have more columns). --- .../suite/binlog/r/load_data_stm_view.result | 22 +++++++++++++++++ .../suite/binlog/t/load_data_stm_view.test | 20 ++++++++++++++++ sql/sql_load.cc | 24 +++++++++---------- 3 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 mysql-test/suite/binlog/r/load_data_stm_view.result create mode 100644 mysql-test/suite/binlog/t/load_data_stm_view.test diff --git a/mysql-test/suite/binlog/r/load_data_stm_view.result b/mysql-test/suite/binlog/r/load_data_stm_view.result new file mode 100644 index 00000000000..39756de8435 --- /dev/null +++ b/mysql-test/suite/binlog/r/load_data_stm_view.result @@ -0,0 +1,22 @@ +create table t1 (i int, j int); +create view v1 as select i from t1; +LOAD DATA LOCAL INFILE 'MYSQLTEST_VARDIR/3940.data' INTO TABLE v1 (i); +LOAD DATA LOCAL INFILE 'MYSQLTEST_VARDIR/3940.data' INTO TABLE v1; +select * from v1; +i +1 +1 +show binlog events from ; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Query # # use `test`; create table t1 (i int, j int) +master-bin.000001 # Query # # use `test`; CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select i from t1 +master-bin.000001 # Query # # BEGIN +master-bin.000001 # Begin_load_query # # ;file_id=#;block_len=# +master-bin.000001 # Execute_load_query # # use `test`; LOAD DATA LOCAL INFILE 'MYSQLTEST_VARDIR/3940.data' IGNORE INTO TABLE `v1` FIELDS TERMINATED BY '\t' ENCLOSED BY '' ESCAPED BY '\\' LINES TERMINATED BY '\n' (`i`) ;file_id=# +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Query # # BEGIN +master-bin.000001 # Begin_load_query # # ;file_id=#;block_len=# +master-bin.000001 # Execute_load_query # # use `test`; LOAD DATA LOCAL INFILE 'MYSQLTEST_VARDIR/3940.data' IGNORE INTO TABLE `v1` FIELDS TERMINATED BY '\t' ENCLOSED BY '' ESCAPED BY '\\' LINES TERMINATED BY '\n' (`i`) ;file_id=# +master-bin.000001 # Query # # COMMIT +drop view v1; +drop table t1; diff --git a/mysql-test/suite/binlog/t/load_data_stm_view.test b/mysql-test/suite/binlog/t/load_data_stm_view.test new file mode 100644 index 00000000000..b70651b4e2d --- /dev/null +++ b/mysql-test/suite/binlog/t/load_data_stm_view.test @@ -0,0 +1,20 @@ +# +# MDEV-3940 Server crash or assertion `item->type() == Item::STRING_ITEM' failure on LOAD DATA through a view with statement binary logging +# + +--source include/have_binlog_format_statement.inc + +--write_file $MYSQLTEST_VARDIR/3940.data +1 +EOF + +create table t1 (i int, j int); +create view v1 as select i from t1; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval LOAD DATA LOCAL INFILE '$MYSQLTEST_VARDIR/3940.data' INTO TABLE v1 (i) +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval LOAD DATA LOCAL INFILE '$MYSQLTEST_VARDIR/3940.data' INTO TABLE v1 +select * from v1; +--source include/show_binlog_events.inc +drop view v1; +drop table t1; diff --git a/sql/sql_load.cc b/sql/sql_load.cc index db0c6da7bca..b4f8b107f9b 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -148,14 +148,8 @@ static int read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, bool ignore_check_option_errors); #ifndef EMBEDDED_LIBRARY -static bool write_execute_load_query_log_event(THD *thd, sql_exchange* ex, - const char* db_arg, /* table's database */ - const char* table_name_arg, - bool is_concurrent, - enum enum_duplicates duplicates, - bool ignore, - bool transactional_table, - int errocode); +static bool write_execute_load_query_log_event(THD *, sql_exchange*, const + char*, const char*, bool, enum enum_duplicates, bool, bool, int); #endif /* EMBEDDED_LIBRARY */ /* @@ -283,9 +277,15 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, if (!fields_vars.elements) { - Field **field; - for (field=table->field; *field ; field++) - fields_vars.push_back(new Item_field(*field)); + Field_iterator_table_ref field_iterator; + field_iterator.set(table_list); + for (; !field_iterator.end_of_fields(); field_iterator.next()) + { + Item *item; + if (!(item= field_iterator.create_item(thd))) + DBUG_RETURN(TRUE); + fields_vars.push_back(item->real_item()); + } bitmap_set_all(table->write_set); table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; /* @@ -737,7 +737,7 @@ static bool write_execute_load_query_log_event(THD *thd, sql_exchange* ex, { if (n++) query_str.append(", "); - if (item->type() == Item::FIELD_ITEM) + if (item->real_type() == Item::FIELD_ITEM) append_identifier(thd, &query_str, item->name, strlen(item->name)); else { From 5d0122bd7719d1b6125af43e29908cb71922e646 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 18 Nov 2014 15:42:40 +0100 Subject: [PATCH 30/36] MDEV-7113 difference between check_vcol_func_processor and check_partition_func_processor MDEV-6789 segfault in Item_func_from_unixtime::get_date on updating table with virtual columns * prohibit VALUES in partitioning expression * prohibit user and system variables in virtual column expressions * fix Item_func_date_format to cache locale (for %M/%W to return the same as MONTHNAME/DAYNAME) * fix Item_func_from_unixtime to cache time_zone directly, not THD (and not to crash) * added tests for other incorrectly allowed (in vcols) functions to see that they don't crash --- mysql-test/r/partition_error.result | 5 ++ mysql-test/suite/vcol/r/not_supported.result | 67 ++++++++++++++++++++ mysql-test/suite/vcol/t/not_supported.test | 58 +++++++++++++++++ mysql-test/t/partition_error.test | 11 ++++ sql/item.h | 1 + sql/item_func.h | 4 +- sql/item_timefunc.cc | 15 +++-- sql/item_timefunc.h | 9 +-- 8 files changed, 155 insertions(+), 15 deletions(-) create mode 100644 mysql-test/suite/vcol/r/not_supported.result create mode 100644 mysql-test/suite/vcol/t/not_supported.test diff --git a/mysql-test/r/partition_error.result b/mysql-test/r/partition_error.result index 851c97c5ec3..0fe1034d447 100644 --- a/mysql-test/r/partition_error.result +++ b/mysql-test/r/partition_error.result @@ -1754,3 +1754,8 @@ PARTITION pmax VALUES LESS THAN MAXVALUE); ERROR HY000: Constant, random or timezone-dependent expressions in (sub)partitioning function are not allowed DROP TABLE t1; End of 5.1 tests +create table t1 (a int) partition by list (values(a) div 1) (partition p0 values in (0), partition p1 values in (1)); +ERROR HY000: This partition function is not allowed +create table t1 (a int) partition by list (uuid_short()) (partition p0 values in (0), partition p1 values in (1)); +ERROR 42000: Constant, random or timezone-dependent expressions in (sub)partitioning function are not allowed near ') (partition p0 values in (0), partition p1 values in (1))' at line 1 +End of 5.5 tests diff --git a/mysql-test/suite/vcol/r/not_supported.result b/mysql-test/suite/vcol/r/not_supported.result new file mode 100644 index 00000000000..06627fccf8b --- /dev/null +++ b/mysql-test/suite/vcol/r/not_supported.result @@ -0,0 +1,67 @@ +set lc_time_names = 'es_MX'; +set time_zone='+10:00'; +set div_precision_increment=20; +create table t1 (a int, b int, v decimal(20,19) as (a/3)); +create table t2 (a int, b int, v int as (a+@a)); +ERROR HY000: Function or expression is not allowed for column 'v' +create table t3 (a int, b int, v int as (a+@@error_count)); +ERROR HY000: Function or expression is not allowed for column 'v' +create table t4 (a int, b int, v int as (@a:=a)); +ERROR HY000: Function or expression is not allowed for column 'v' +create table t5 (a int, b int, v varchar(100) as (monthname(a))); +create table t6 (a int, b int, v varchar(100) as (dayname(a))); +create table t7 (a int, b int, v varchar(100) as (date_format(a, '%W %a %M %b'))); +create table t8 (a int, b int, v varchar(100) as (from_unixtime(a))); +insert t1 (a,b) values (1,2); +insert t5 (a,b) values (20141010,2); +insert t6 (a,b) values (20141010,2); +insert t7 (a,b) values (20141010,2); +insert t8 (a,b) values (1234567890,2); +select * from t1; +a b v +1 2 0.3333333333333333333 +select * from t5; +a b v +20141010 2 octubre +select * from t6; +a b v +20141010 2 viernes +select * from t7; +a b v +20141010 2 viernes vie octubre oct +select * from t8; +a b v +1234567890 2 2009-02-14 09:31:30 +set time_zone='+1:00'; +select * from t1; +a b v +1 2 0.3333333333333333333 +select * from t5; +a b v +20141010 2 octubre +select * from t6; +a b v +20141010 2 viernes +select * from t7; +a b v +20141010 2 viernes vie octubre oct +select * from t8; +a b v +1234567890 2 2009-02-14 09:31:30 +flush tables; +select * from t1; +a b v +1 2 0.3333333330000000000 +select * from t5; +a b v +20141010 2 October +select * from t6; +a b v +20141010 2 Friday +select * from t7; +a b v +20141010 2 Friday Fri October Oct +select * from t8; +a b v +1234567890 2 2009-02-14 00:31:30 +drop table t1, t5, t6, t7, t8; diff --git a/mysql-test/suite/vcol/t/not_supported.test b/mysql-test/suite/vcol/t/not_supported.test new file mode 100644 index 00000000000..70b9dea69fd --- /dev/null +++ b/mysql-test/suite/vcol/t/not_supported.test @@ -0,0 +1,58 @@ +# +# MDEV-7113 difference between check_vcol_func_processor and check_partition_func_processor +# + +# the following functions must not be supported in virtual columns. +# but for compatibility reasons it won't be done in a GA version, +# we'll only fix most critical issues (inconsistent results, crashes) + +connect (con1, localhost, root); + +set lc_time_names = 'es_MX'; +set time_zone='+10:00'; +set div_precision_increment=20; + +create table t1 (a int, b int, v decimal(20,19) as (a/3)); +--error ER_VIRTUAL_COLUMN_FUNCTION_IS_NOT_ALLOWED +create table t2 (a int, b int, v int as (a+@a)); +--error ER_VIRTUAL_COLUMN_FUNCTION_IS_NOT_ALLOWED +create table t3 (a int, b int, v int as (a+@@error_count)); +--error ER_VIRTUAL_COLUMN_FUNCTION_IS_NOT_ALLOWED +create table t4 (a int, b int, v int as (@a:=a)); +create table t5 (a int, b int, v varchar(100) as (monthname(a))); +create table t6 (a int, b int, v varchar(100) as (dayname(a))); +create table t7 (a int, b int, v varchar(100) as (date_format(a, '%W %a %M %b'))); +create table t8 (a int, b int, v varchar(100) as (from_unixtime(a))); + +insert t1 (a,b) values (1,2); +insert t5 (a,b) values (20141010,2); +insert t6 (a,b) values (20141010,2); +insert t7 (a,b) values (20141010,2); +insert t8 (a,b) values (1234567890,2); + +select * from t1; +select * from t5; +select * from t6; +select * from t7; +select * from t8; + +disconnect con1; +connection default; +set time_zone='+1:00'; + +select * from t1; +select * from t5; +select * from t6; +select * from t7; +select * from t8; + +flush tables; + +select * from t1; +select * from t5; +select * from t6; +select * from t7; +select * from t8; + +drop table t1, t5, t6, t7, t8; + diff --git a/mysql-test/t/partition_error.test b/mysql-test/t/partition_error.test index f8f09e4b5e7..64799574140 100644 --- a/mysql-test/t/partition_error.test +++ b/mysql-test/t/partition_error.test @@ -2005,3 +2005,14 @@ PARTITION pmax VALUES LESS THAN MAXVALUE); DROP TABLE t1; --echo End of 5.1 tests + +# +# MDEV-7113 difference between check_vcol_func_processor and check_partition_func_processor +# +--error ER_PARTITION_FUNCTION_IS_NOT_ALLOWED +create table t1 (a int) partition by list (values(a) div 1) (partition p0 values in (0), partition p1 values in (1)); + +--error ER_PARSE_ERROR +create table t1 (a int) partition by list (uuid_short()) (partition p0 values in (0), partition p1 values in (1)); + +--echo End of 5.5 tests diff --git a/sql/item.h b/sql/item.h index f7ee860390f..57e91e5b3d8 100644 --- a/sql/item.h +++ b/sql/item.h @@ -3939,6 +3939,7 @@ public: return arg->walk(processor, walk_subquery, args) || (this->*processor)(args); } + bool check_partition_func_processor(uchar *int_arg) {return TRUE;} bool check_vcol_func_processor(uchar *arg) { return trace_unsupported_by_check_vcol_func_processor("values"); diff --git a/sql/item_func.h b/sql/item_func.h index 49966964fbb..ab0ae5f0bda 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1696,6 +1696,7 @@ public: bool register_field_in_bitmap(uchar *arg); bool set_entry(THD *thd, bool create_if_not_exists); void cleanup(); + bool check_vcol_func_processor(uchar *int_arg) {return TRUE;} }; @@ -1735,6 +1736,7 @@ public: { return this; } + bool check_vcol_func_processor(uchar *int_arg) { return TRUE;} }; @@ -1817,6 +1819,7 @@ public: bool eq(const Item *item, bool binary_cmp) const; void cleanup(); + bool check_vcol_func_processor(uchar *int_arg) { return TRUE;} }; @@ -2090,7 +2093,6 @@ public: longlong val_int(); void fix_length_and_dec() { max_length= 21; unsigned_flag=1; } - bool check_partition_func_processor(uchar *int_arg) {return FALSE;} bool check_vcol_func_processor(uchar *int_arg) { return trace_unsupported_by_check_vcol_func_processor(func_name()); diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 15701658015..59a5f1849f8 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -450,16 +450,14 @@ err: Create a formated date/time value in a string. */ -bool make_date_time(DATE_TIME_FORMAT *format, MYSQL_TIME *l_time, - timestamp_type type, String *str) +static bool make_date_time(DATE_TIME_FORMAT *format, MYSQL_TIME *l_time, + timestamp_type type, MY_LOCALE *locale, String *str) { char intbuff[15]; uint hours_i; uint weekday; ulong length; const char *ptr, *end; - THD *thd= current_thd; - MY_LOCALE *locale= thd->variables.lc_time_names; str->length(0); @@ -1726,6 +1724,8 @@ overflow: void Item_func_date_format::fix_length_and_dec() { THD* thd= current_thd; + locale= thd->variables.lc_time_names; + /* Must use this_item() in case it's a local SP variable (for ->max_length and ->str_value) @@ -1889,7 +1889,7 @@ String *Item_func_date_format::val_str(String *str) if (!make_date_time(&date_time_format, &l_time, is_time_format ? MYSQL_TIMESTAMP_TIME : MYSQL_TIMESTAMP_DATE, - str)) + locale, str)) return str; null_date: @@ -1900,8 +1900,9 @@ null_date: void Item_func_from_unixtime::fix_length_and_dec() { - thd= current_thd; + THD *thd= current_thd; thd->time_zone_used= 1; + tz= thd->variables.time_zone; decimals= args[0]->decimals; Item_temporal_func::fix_length_and_dec(); } @@ -1922,7 +1923,7 @@ bool Item_func_from_unixtime::get_date(MYSQL_TIME *ltime, if (args[0]->null_value || sign || sec > TIMESTAMP_MAX_VALUE) return (null_value= 1); - thd->variables.time_zone->gmt_sec_to_TIME(ltime, (my_time_t)sec); + tz->gmt_sec_to_TIME(ltime, (my_time_t)sec); ltime->second_part= sec_part; diff --git a/sql/item_timefunc.h b/sql/item_timefunc.h index 455540c4b0d..3a03ee4b27a 100644 --- a/sql/item_timefunc.h +++ b/sql/item_timefunc.h @@ -688,6 +688,7 @@ public: class Item_func_date_format :public Item_str_func { + MY_LOCALE *locale; int fixed_length; const bool is_time_format; String value; @@ -705,7 +706,7 @@ public: class Item_func_from_unixtime :public Item_temporal_func { - THD *thd; + Time_zone *tz; public: Item_func_from_unixtime(Item *a) :Item_temporal_func(a) {} const char *func_name() const { return "from_unixtime"; } @@ -1046,10 +1047,4 @@ public: bool get_date(MYSQL_TIME *res, ulonglong fuzzy_date); }; - -/* Function prototypes */ - -bool make_date_time(DATE_TIME_FORMAT *format, MYSQL_TIME *l_time, - timestamp_type type, String *str); - #endif /* ITEM_TIMEFUNC_INCLUDED */ From cc2c296309def442de0ebbe101f1fd61cb1acc0a Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 18 Nov 2014 15:42:48 +0100 Subject: [PATCH 31/36] MDEV-4513 Valgrind warnings (Conditional jump or move depends on uninitialised value) in inflate on UNCOMPRESS --- mysql-test/r/func_compress.result | 8 ++++++++ mysql-test/t/func_compress.test | 9 +++++++++ sql/item_strfunc.cc | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/func_compress.result b/mysql-test/r/func_compress.result index 22b75de8c60..9fde006b377 100644 --- a/mysql-test/r/func_compress.result +++ b/mysql-test/r/func_compress.result @@ -147,3 +147,11 @@ DROP TABLE t1; # # End of 5.3 tests # +SELECT UNCOMPRESS(CAST(0 AS BINARY(5))); +UNCOMPRESS(CAST(0 AS BINARY(5))) +NULL +Warnings: +Warning 1259 ZLIB: Input data corrupted +# +# End of 5.5 tests +# diff --git a/mysql-test/t/func_compress.test b/mysql-test/t/func_compress.test index eaed0c88fe1..fc3d2697426 100644 --- a/mysql-test/t/func_compress.test +++ b/mysql-test/t/func_compress.test @@ -136,3 +136,12 @@ DROP TABLE t1; --echo # --echo # End of 5.3 tests --echo # + +# +# MDEV-4513 Valgrind warnings (Conditional jump or move depends on uninitialised value) in inflate on UNCOMPRESS +# +SELECT UNCOMPRESS(CAST(0 AS BINARY(5))); + +--echo # +--echo # End of 5.5 tests +--echo # diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index e7e73794771..eedf1499403 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -3720,7 +3720,7 @@ String *Item_func_uncompress::val_str(String *str) goto err; if ((err= uncompress((Byte*)buffer.ptr(), &new_size, - ((const Bytef*)res->ptr())+4,res->length())) == Z_OK) + ((const Bytef*)res->ptr())+4,res->length()-4)) == Z_OK) { buffer.length((uint32) new_size); return &buffer; From f3a800ef6edf8f2c1fa367aefbe7b6567c68a919 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 18 Nov 2014 15:43:01 +0100 Subject: [PATCH 32/36] MDEV-7028 mysql_config produces invalid cflags (was: udf_example.c couldn't compile) two bugs in mysql_config: 1. flags like -Werror=format were not stipped out 2. one-pass s/// command was used that could not remove all matching flags (e.g. s/ A */ /g cannot remove all A's in " 1 2 A A A A A 4 5 ") --- scripts/mysql_config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mysql_config.sh b/scripts/mysql_config.sh index e4640961b34..57bdb4cbd56 100644 --- a/scripts/mysql_config.sh +++ b/scripts/mysql_config.sh @@ -136,12 +136,12 @@ cflags="$include @CFLAGS@ " #note: end space! for remove in DDBUG_OFF DSAFE_MUTEX DUNIV_MUST_NOT_INLINE DFORCE_INIT_OF_VARS \ DEXTRA_DEBUG DHAVE_valgrind O 'O[0-9]' 'xO[0-9]' 'W[-A-Za-z]*' \ 'mtune=[-A-Za-z0-9]*' 'mcpu=[-A-Za-z0-9]*' 'march=[-A-Za-z0-9]*' \ - Xa xstrconst "xc99=none" AC99 \ + Xa xstrconst "xc99=none" AC99 'W[-A-Za-z]*=[-A-Za-z0-9]*' \ unroll2 ip mp restrict do # The first option we might strip will always have a space before it because # we set -I$pkgincludedir as the first option - cflags=`echo "$cflags"|sed -e "s/ -$remove */ /g"` + cflags=`echo "$cflags"|sed -e ':again' -e "s/ -$remove */ /g" -e 't again'` done cflags=`echo "$cflags"|sed -e 's/ *\$//'` From c8fa6f782c9e3bc7ac83b36e486e9177fa9fa18c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 18 Nov 2014 17:56:49 +0100 Subject: [PATCH 33/36] improve OpenSSL error reporting e.g. from "error:00000001:lib(0):func(0):reason(1)" to "error:140830B5:SSL routines:SSL3_CLIENT_HELLO:no ciphers available" --- vio/viossl.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vio/viossl.c b/vio/viossl.c index 60e663cacaf..fd5dfc51f17 100644 --- a/vio/viossl.c +++ b/vio/viossl.c @@ -26,6 +26,18 @@ #ifdef HAVE_OPENSSL +#ifndef HAVE_YASSL +/* + yassl seem to be different here, SSL_get_error() value can be + directly passed to ERR_error_string(), and these errors don't go + into ERR_get_error() stack. + in openssl, apparently, SSL_get_error() values live in a different + namespace, one needs to use ERR_get_error() as an argument + for ERR_error_string(). +*/ +#define SSL_get_error(X,Y) ERR_get_error() +#endif + #ifndef DBUG_OFF static void From 386e2e52f63971d3c1f33fb6b90715549d81ed1e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 18 Nov 2014 17:56:58 +0100 Subject: [PATCH 34/36] new mysqltest connect option SSL-CIPHER=xxxx --- client/mysqltest.cc | 8 +++++++- mysql-test/r/openssl_1.result | 2 ++ mysql-test/t/openssl_1.test | 13 ++++++++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index ddfb52539c4..6ead479b94e 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -5883,6 +5883,7 @@ void do_connect(struct st_command *command) { int con_port= opt_port; char *con_options; + char *ssl_cipher= 0; my_bool con_ssl= 0, con_compress= 0; my_bool con_pipe= 0; my_bool con_shm __attribute__ ((unused))= 0; @@ -5971,6 +5972,11 @@ void do_connect(struct st_command *command) length= (size_t) (end - con_options); if (length == 3 && !strncmp(con_options, "SSL", 3)) con_ssl= 1; + else if (!strncmp(con_options, "SSL-CIPHER=", 11)) + { + con_ssl= 1; + ssl_cipher=con_options + 11; + } else if (length == 8 && !strncmp(con_options, "COMPRESS", 8)) con_compress= 1; else if (length == 4 && !strncmp(con_options, "PIPE", 4)) @@ -6027,7 +6033,7 @@ void do_connect(struct st_command *command) { #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY) mysql_ssl_set(con_slot->mysql, opt_ssl_key, opt_ssl_cert, opt_ssl_ca, - opt_ssl_capath, opt_ssl_cipher); + opt_ssl_capath, ssl_cipher ? ssl_cipher : opt_ssl_cipher); #if MYSQL_VERSION_ID >= 50000 /* Turn on ssl_verify_server_cert only if host is "localhost" */ opt_ssl_verify_server_cert= !strcmp(ds_host.str, "localhost"); diff --git a/mysql-test/r/openssl_1.result b/mysql-test/r/openssl_1.result index 76b8e887d89..4627f03a8a3 100644 --- a/mysql-test/r/openssl_1.result +++ b/mysql-test/r/openssl_1.result @@ -7,6 +7,8 @@ grant select on test.* to ssl_user3@localhost require cipher "DHE-RSA-AES256-SHA grant select on test.* to ssl_user4@localhost require cipher "DHE-RSA-AES256-SHA" AND SUBJECT "/C=SE/ST=Uppsala/O=MySQL AB" ISSUER "/C=SE/ST=Uppsala/L=Uppsala/O=MySQL AB"; grant select on test.* to ssl_user5@localhost require cipher "DHE-RSA-AES256-SHA" AND SUBJECT "xxx"; flush privileges; +connect(localhost,ssl_user2,,test,MASTER_PORT,MASTER_SOCKET); +ERROR 28000: Access denied for user 'ssl_user2'@'localhost' (using password: NO) connect(localhost,ssl_user5,,test,MASTER_PORT,MASTER_SOCKET); ERROR 28000: Access denied for user 'ssl_user5'@'localhost' (using password: NO) SHOW STATUS LIKE 'Ssl_cipher'; diff --git a/mysql-test/t/openssl_1.test b/mysql-test/t/openssl_1.test index 0c8f81e4712..24980269a24 100644 --- a/mysql-test/t/openssl_1.test +++ b/mysql-test/t/openssl_1.test @@ -20,13 +20,16 @@ grant select on test.* to ssl_user4@localhost require cipher "DHE-RSA-AES256-SHA grant select on test.* to ssl_user5@localhost require cipher "DHE-RSA-AES256-SHA" AND SUBJECT "xxx"; flush privileges; -connect (con1,localhost,ssl_user1,,,,,SSL); -connect (con2,localhost,ssl_user2,,,,,SSL); -connect (con3,localhost,ssl_user3,,,,,SSL); -connect (con4,localhost,ssl_user4,,,,,SSL); +connect (con1,localhost,ssl_user1,,,,,SSL-CIPHER=DHE-RSA-AES256-SHA); --replace_result $MASTER_MYSOCK MASTER_SOCKET $MASTER_MYPORT MASTER_PORT --error ER_ACCESS_DENIED_ERROR -connect (con5,localhost,ssl_user5,,,,,SSL); +connect (con2,localhost,ssl_user2,,,,,SSL-CIPHER=RC4-SHA); +connect (con2,localhost,ssl_user2,,,,,SSL-CIPHER=DHE-RSA-AES256-SHA); +connect (con3,localhost,ssl_user3,,,,,SSL-CIPHER=DHE-RSA-AES256-SHA); +connect (con4,localhost,ssl_user4,,,,,SSL-CIPHER=DHE-RSA-AES256-SHA); +--replace_result $MASTER_MYSOCK MASTER_SOCKET $MASTER_MYPORT MASTER_PORT +--error ER_ACCESS_DENIED_ERROR +connect (con5,localhost,ssl_user5,,,,,SSL-CIPHER=DHE-RSA-AES256-SHA); connection con1; # Check ssl turned on From cb8f837a3d0d16bdbfc9c545d32527ad28b0d841 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 18 Nov 2014 17:57:06 +0100 Subject: [PATCH 35/36] MDEV-6975 Implement TLS protocol change SSL methods to be SSLv23 (according to openssl manpage: "A TLS/SSL connection established with these methods may understand the SSLv2, SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols") from TLSv1 methods, that go back to the initial SSL implementation in MySQL in 2001. OpenSSL default ciphers are different if TLSv1.2 is enabled, so tests need to take this into account. --- mysql-test/mysql-test-run.pl | 2 + mysql-test/r/openssl-poodle_6975,sslv3.result | 25 ++++++++++++ .../r/openssl-poodle_6975,tlsv12.result | 25 ++++++++++++ mysql-test/t/openssl-poodle_6975.combinations | 6 +++ mysql-test/t/openssl-poodle_6975.test | 38 +++++++++++++++++++ mysql-test/t/openssl_1.test | 2 + mysql-test/t/ssl.test | 2 + mysql-test/t/ssl_8k_key-master.opt | 4 +- mysql-test/t/ssl_compress.test | 2 + vio/viosslfactories.c | 4 +- 10 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 mysql-test/r/openssl-poodle_6975,sslv3.result create mode 100644 mysql-test/r/openssl-poodle_6975,tlsv12.result create mode 100644 mysql-test/t/openssl-poodle_6975.combinations create mode 100644 mysql-test/t/openssl-poodle_6975.test diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index ddfed62c238..60e0399efd4 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -4801,6 +4801,8 @@ sub extract_warning_lines ($$) { qr|feedback plugin: failed to retrieve the MAC address|, qr|Plugin 'FEEDBACK' init function returned error|, qr|Plugin 'FEEDBACK' registration as a INFORMATION SCHEMA failed|, + qr|Failed to setup SSL|, + qr|SSL error: Failed to set ciphers to use|, ); my $matched_lines= []; diff --git a/mysql-test/r/openssl-poodle_6975,sslv3.result b/mysql-test/r/openssl-poodle_6975,sslv3.result new file mode 100644 index 00000000000..52d5978749e --- /dev/null +++ b/mysql-test/r/openssl-poodle_6975,sslv3.result @@ -0,0 +1,25 @@ +grant select on test.* to ssl_sslv3@localhost require cipher "RC4-SHA"; +grant select on test.* to ssl_tls12@localhost require cipher "AES128-SHA256"; +TLS1.2 ciphers: user is ok with any cipher +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +TLS1.2 ciphers: user requires SSLv3 cipher RC4-SHA +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +TLS1.2 ciphers: user requires TLSv1.2 cipher AES128-SHA256 +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +SSLv3 ciphers: user is ok with any cipher +Variable_name Value +Ssl_cipher RC4-SHA +Variable_name Value +Ssl_cipher DHE-RSA-AES256-SHA +SSLv3 ciphers: user requires SSLv3 cipher RC4-SHA +Variable_name Value +Ssl_cipher RC4-SHA +ERROR 1045 (28000): Access denied for user 'ssl_sslv3'@'localhost' (using password: NO) +SSLv3 ciphers: user requires TLSv1.2 cipher AES128-SHA256 +ERROR 1045 (28000): Access denied for user 'ssl_tls12'@'localhost' (using password: NO) +ERROR 1045 (28000): Access denied for user 'ssl_tls12'@'localhost' (using password: NO) +drop user ssl_sslv3@localhost; +drop user ssl_tls12@localhost; diff --git a/mysql-test/r/openssl-poodle_6975,tlsv12.result b/mysql-test/r/openssl-poodle_6975,tlsv12.result new file mode 100644 index 00000000000..033220427be --- /dev/null +++ b/mysql-test/r/openssl-poodle_6975,tlsv12.result @@ -0,0 +1,25 @@ +grant select on test.* to ssl_sslv3@localhost require cipher "RC4-SHA"; +grant select on test.* to ssl_tls12@localhost require cipher "AES128-SHA256"; +TLS1.2 ciphers: user is ok with any cipher +Variable_name Value +Ssl_cipher AES128-SHA256 +Variable_name Value +Ssl_cipher DHE-RSA-AES256-GCM-SHA384 +TLS1.2 ciphers: user requires SSLv3 cipher RC4-SHA +ERROR 1045 (28000): Access denied for user 'ssl_sslv3'@'localhost' (using password: NO) +ERROR 1045 (28000): Access denied for user 'ssl_sslv3'@'localhost' (using password: NO) +TLS1.2 ciphers: user requires TLSv1.2 cipher AES128-SHA256 +Variable_name Value +Ssl_cipher AES128-SHA256 +ERROR 1045 (28000): Access denied for user 'ssl_tls12'@'localhost' (using password: NO) +SSLv3 ciphers: user is ok with any cipher +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +SSLv3 ciphers: user requires SSLv3 cipher RC4-SHA +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +SSLv3 ciphers: user requires TLSv1.2 cipher AES128-SHA256 +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +ERROR 2026 (HY000): SSL connection error: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure +drop user ssl_sslv3@localhost; +drop user ssl_tls12@localhost; diff --git a/mysql-test/t/openssl-poodle_6975.combinations b/mysql-test/t/openssl-poodle_6975.combinations new file mode 100644 index 00000000000..3147de1cb63 --- /dev/null +++ b/mysql-test/t/openssl-poodle_6975.combinations @@ -0,0 +1,6 @@ +[tlsv12] +loose-ssl-cipher=TLSv1.2 + +[sslv3] +loose-ssl-cipher=SSLv3 + diff --git a/mysql-test/t/openssl-poodle_6975.test b/mysql-test/t/openssl-poodle_6975.test new file mode 100644 index 00000000000..bc6397c5c28 --- /dev/null +++ b/mysql-test/t/openssl-poodle_6975.test @@ -0,0 +1,38 @@ +# +# MDEV-6975 Implement TLS protocol +# +# test SSLv3 and TLSv1.2 ciphers when OpenSSL is restricted to SSLv3 or TLSv1.2 +# +source include/have_ssl_communication.inc; + +# this is OpenSSL test. + +grant select on test.* to ssl_sslv3@localhost require cipher "RC4-SHA"; +grant select on test.* to ssl_tls12@localhost require cipher "AES128-SHA256"; + +let $mysql=$MYSQL --ssl-key=$MYSQL_TEST_DIR/std_data/client-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem -e "SHOW STATUS LIKE 'ssl_Cipher'" 2>&1; + +disable_abort_on_error; +echo TLS1.2 ciphers: user is ok with any cipher; +exec $mysql --ssl-cipher=AES128-SHA256; +exec $mysql --ssl-cipher=TLSv1.2; +echo TLS1.2 ciphers: user requires SSLv3 cipher RC4-SHA; +exec $mysql --user ssl_sslv3 --ssl-cipher=AES128-SHA256; +exec $mysql --user ssl_sslv3 --ssl-cipher=TLSv1.2; +echo TLS1.2 ciphers: user requires TLSv1.2 cipher AES128-SHA256; +exec $mysql --user ssl_tls12 --ssl-cipher=AES128-SHA256; +exec $mysql --user ssl_tls12 --ssl-cipher=TLSv1.2; + +echo SSLv3 ciphers: user is ok with any cipher; +exec $mysql --ssl-cipher=RC4-SHA; +exec $mysql --ssl-cipher=SSLv3; +echo SSLv3 ciphers: user requires SSLv3 cipher RC4-SHA; +exec $mysql --user ssl_sslv3 --ssl-cipher=RC4-SHA; +exec $mysql --user ssl_sslv3 --ssl-cipher=SSLv3; +echo SSLv3 ciphers: user requires TLSv1.2 cipher AES128-SHA256; +exec $mysql --user ssl_tls12 --ssl-cipher=RC4-SHA; +exec $mysql --user ssl_tls12 --ssl-cipher=SSLv3; + +drop user ssl_sslv3@localhost; +drop user ssl_tls12@localhost; + diff --git a/mysql-test/t/openssl_1.test b/mysql-test/t/openssl_1.test index 24980269a24..bee0e2cc720 100644 --- a/mysql-test/t/openssl_1.test +++ b/mysql-test/t/openssl_1.test @@ -132,6 +132,7 @@ drop table t1; # verification of servers certificate by setting both ca certificate # and ca path to NULL # +--replace_result DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-SHA --exec $MYSQL --ssl --ssl-key=$MYSQL_TEST_DIR/std_data/client-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem -e "SHOW STATUS LIKE 'ssl_Cipher'" 2>&1 --echo End of 5.0 tests @@ -258,6 +259,7 @@ select 'is still running; no cipher request crashed the server' as result from d GRANT SELECT ON test.* TO bug42158@localhost REQUIRE X509; FLUSH PRIVILEGES; connect(con1,localhost,bug42158,,,,,SSL); +--replace_result DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-SHA SHOW STATUS LIKE 'Ssl_cipher'; disconnect con1; connection default; diff --git a/mysql-test/t/ssl.test b/mysql-test/t/ssl.test index 341ec0591e7..d6f88fe888e 100644 --- a/mysql-test/t/ssl.test +++ b/mysql-test/t/ssl.test @@ -11,12 +11,14 @@ connect (ssl_con,localhost,root,,,,,SSL); # Check ssl turned on +--replace_result DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-SHA SHOW STATUS LIKE 'Ssl_cipher'; # Source select test case -- source include/common-tests.inc # Check ssl turned on +--replace_result DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-SHA SHOW STATUS LIKE 'Ssl_cipher'; connection default; diff --git a/mysql-test/t/ssl_8k_key-master.opt b/mysql-test/t/ssl_8k_key-master.opt index b58ca7f39f0..531c0abc9f1 100644 --- a/mysql-test/t/ssl_8k_key-master.opt +++ b/mysql-test/t/ssl_8k_key-master.opt @@ -1 +1,3 @@ ---loose-ssl-key=$MYSQL_TEST_DIR/std_data/server8k-key.pem --loose-ssl-cert=$MYSQL_TEST_DIR/std_data/server8k-cert.pem +--loose-ssl-key=$MYSQL_TEST_DIR/std_data/server8k-key.pem +--loose-ssl-cert=$MYSQL_TEST_DIR/std_data/server8k-cert.pem +--loose-ssl-cipher=DHE-RSA-AES256-SHA diff --git a/mysql-test/t/ssl_compress.test b/mysql-test/t/ssl_compress.test index 8e0dea53fe7..5e45e3824a2 100644 --- a/mysql-test/t/ssl_compress.test +++ b/mysql-test/t/ssl_compress.test @@ -11,6 +11,7 @@ connect (ssl_compress_con,localhost,root,,,,,SSL COMPRESS); # Check ssl turned on +--replace_result DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-SHA SHOW STATUS LIKE 'Ssl_cipher'; # Check compression turned on @@ -20,6 +21,7 @@ SHOW STATUS LIKE 'Compression'; -- source include/common-tests.inc # Check ssl turned on +--replace_result DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-SHA SHOW STATUS LIKE 'Ssl_cipher'; # Check compression turned on diff --git a/vio/viosslfactories.c b/vio/viosslfactories.c index 3fde307e4dd..c64bba338a4 100644 --- a/vio/viosslfactories.c +++ b/vio/viosslfactories.c @@ -190,8 +190,8 @@ new_VioSSLFd(const char *key_file, const char *cert_file, DBUG_RETURN(0); if (!(ssl_fd->ssl_context= SSL_CTX_new(is_client_method ? - TLSv1_client_method() : - TLSv1_server_method()))) + SSLv23_client_method() : + SSLv23_server_method()))) { *error= SSL_INITERR_MEMFAIL; DBUG_PRINT("error", ("%s", sslGetErrString(*error))); From 496fda66fdc34b447ef4dec26d1250b034a321e3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 19 Nov 2014 00:19:52 +0100 Subject: [PATCH 36/36] openssl-poodle_6975.test: don't run it for older OpenSSL versions --- mysql-test/suite.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm index ef0325bd3e7..60d3c0b6d8d 100644 --- a/mysql-test/suite.pm +++ b/mysql-test/suite.pm @@ -39,6 +39,9 @@ sub skip_combinations { } $skip{'include/check_ipv6.inc'} = 'No IPv6' unless ipv6_ok(); + $skip{'t/openssl-poodle_6975.test'} = 'no or too old openssl' + unless ! IS_WINDOWS and ! system "openssl ciphers TLSv1.2 2>&1 >/dev/null"; + %skip; }