Merge 10.2 into 10.3

cmake/submodules.cmake

@@ -1,4 +1,10 @@
 # update submodules automatically
+
+OPTION(UPDATE_SUBMODULES "Update submodules automatically" ON)
+IF(NOT UPDATE_SUBMODULES)
+  RETURN()
+ENDIF()
+
 IF(GIT_EXECUTABLE AND EXISTS "${CMAKE_SOURCE_DIR}/.git")
   EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" config --get cmake.update-submodules
                   WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"

include/mysql/psi/mysql_socket.h

@@ -735,7 +735,7 @@ inline_mysql_socket_send
  MYSQL_SOCKET mysql_socket, const SOCKBUF_T *buf, size_t n, int flags)
 {
   ssize_t result;
-
+  DBUG_ASSERT(mysql_socket.fd != INVALID_SOCKET);
 #ifdef HAVE_PSI_SOCKET_INTERFACE
   if (psi_likely(mysql_socket.m_psi != NULL))
   {
@@ -776,7 +776,7 @@ inline_mysql_socket_recv
  MYSQL_SOCKET mysql_socket,  SOCKBUF_T *buf, size_t n, int flags)
 {
   ssize_t result;
-
+  DBUG_ASSERT(mysql_socket.fd != INVALID_SOCKET);
 #ifdef HAVE_PSI_SOCKET_INTERFACE
   if (psi_likely(mysql_socket.m_psi != NULL))
   {

libmariadb

@@ -1 +1 @@
-Subproject commit 17d0e5317604718fc059ee1ad4bd2ea36494226b
+Subproject commit 99f383c85c952287f8d3db927665061cd226e0f7

mysql-test/suite/innodb/r/innodb-index,debug.rdiff

@@ -0,0 +1,10 @@
+--- innodb-index.result
++++ innodb-index.reject
+@@ -1851,6 +1851,7 @@
+ #
+ # MDEV-15325 Incomplete validation of missing tablespace during recovery
+ #
++SET GLOBAL DEBUG_DBUG='+d,fil_names_write_bogus';
+ CREATE TABLE t1(f1 INT PRIMARY KEY)ENGINE=InnoDB;
+ CREATE TABLE t2(f1 INT PRIMARY KEY)ENGINE=InnoDB;
+ # Kill the server

mysql-test/suite/innodb/t/innodb-index.test

@@ -1,6 +1,7 @@
 -- source include/have_innodb.inc
 # Embedded server tests do not support restarting.
 -- source include/not_embedded.inc
+-- source include/maybe_debug.inc
 
 let $MYSQLD_DATADIR= `select @@datadir`;
 
@@ -1084,6 +1085,9 @@ drop table t1;
 --echo #
 
 --source include/no_checkpoint_start.inc
+if ($have_debug) {
+SET GLOBAL DEBUG_DBUG='+d,fil_names_write_bogus';
+}
 CREATE TABLE t1(f1 INT PRIMARY KEY)ENGINE=InnoDB;
 
 CREATE TABLE t2(f1 INT PRIMARY KEY)ENGINE=InnoDB;

mysql-test/suite/innodb_fts/r/crash_recovery.result

@@ -1,3 +1,4 @@
+FLUSH TABLES;
 CREATE TABLE articles (
 id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
 title VARCHAR(200),

mysql-test/suite/innodb_fts/t/crash_recovery.test

@@ -7,6 +7,7 @@
 # The embedded server tests do not support restarting.
 --source include/not_embedded.inc
 
+FLUSH TABLES;
 # Following are test for crash recovery on FTS index, the first scenario
 # is for bug Bug #14586855 INNODB: FAILING ASSERTION: (DICT_INDEX_GET_N_UNIQUE(
 # PLAN->INDEX) <= PLAN->N_EXAC

plugin/aws_key_management/aws_key_management_plugin.cc

@@ -106,6 +106,14 @@ static std::mutex mtx;
 
 static Aws::KMS::KMSClient *client;
 
+static void print_kms_error(const char *func, const Aws::Client::AWSError<Aws::KMS::KMSErrors>& err)
+{
+  my_printf_error(ER_UNKNOWN_ERROR,
+    "AWS KMS plugin : KMS Client API '%s' failed : %s - %s",
+    ME_ERROR_LOG,
+    func, err.GetExceptionName().c_str(), err.GetMessage().c_str());
+}
+
 #if WITH_AWS_MOCK
 /* 
   Mock routines to test plugin without actual AWS KMS interaction
@@ -127,7 +135,7 @@ static int mock_generate_encrypted_key(Aws::Utils::ByteBuffer *result)
 }
 
 
-static int mock_decrypt(Aws::Utils::ByteBuffer input, Aws::Utils::ByteBuffer* output, Aws::String *errmsg)
+static int mock_decrypt(Aws::Utils::ByteBuffer input, Aws::Utils::ByteBuffer* output)
 {
   /* We do not encrypt or decrypt in mock mode.*/
   *output = input;
@@ -401,14 +409,14 @@ static unsigned int get_latest_key_version_nolock(unsigned int key_id)
 }
 
 /* Decrypt Byte buffer with AWS. */
-static int aws_decrypt(Aws::Utils::ByteBuffer input, Aws::Utils::ByteBuffer* output, Aws::String *errmsg)
+static int aws_decrypt(Aws::Utils::ByteBuffer input, Aws::Utils::ByteBuffer* output)
 {
   DecryptRequest request;
   request.SetCiphertextBlob(input);
   DecryptOutcome outcome = client->Decrypt(request);
   if (!outcome.IsSuccess())
   {
-    *errmsg = outcome.GetError().GetMessage();
+    print_kms_error("Decrypt", outcome.GetError());
     return -1;
   }
   *output= outcome.GetResult().GetPlaintext();
@@ -416,13 +424,13 @@ static int aws_decrypt(Aws::Utils::ByteBuffer input, Aws::Utils::ByteBuffer* out
 }
 
 
-static int decrypt(Aws::Utils::ByteBuffer input, Aws::Utils::ByteBuffer* output, Aws::String *errmsg)
+static int decrypt(Aws::Utils::ByteBuffer input, Aws::Utils::ByteBuffer* output)
 {
 #if WITH_AWS_MOCK
   if(mock)
-    return mock_decrypt(input,output, errmsg);
+    return mock_decrypt(input,output);
 #endif
-  return aws_decrypt(input, output, errmsg);
+  return aws_decrypt(input, output);
 }
 
 /* 
@@ -452,12 +460,9 @@ static  int read_and_decrypt_key(const char *path, KEY_INFO *info)
 
   Aws::Utils::ByteBuffer input((unsigned char *)contents.data(), pos);
   Aws::Utils::ByteBuffer plaintext;
-  Aws::String errmsg;
 
-  if (decrypt(input, &plaintext, &errmsg))
+  if (decrypt(input, &plaintext))
   {
-      my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: Decrypt failed for %s : %s", ME_ERROR_LOG, path,
-      errmsg.c_str());
       return -1;
   }
 
@@ -491,9 +496,7 @@ int aws_generate_encrypted_key(Aws::Utils::ByteBuffer *result)
   outcome= client->GenerateDataKeyWithoutPlaintext(request);
   if (!outcome.IsSuccess())
   {
-    my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin : GenerateDataKeyWithoutPlaintext failed : %s - %s", ME_ERROR_LOG,
+    print_kms_error("GenerateDataKeyWithoutPlaintext", outcome.GetError());
-      outcome.GetError().GetExceptionName().c_str(),
-      outcome.GetError().GetMessage().c_str());
     return(-1);
   }
  *result =  outcome.GetResult().GetCiphertextBlob();

sql/item_strfunc.cc

@@ -603,7 +603,7 @@ String *Item_func_concat::val_str(String *str)
     goto null;
 
   if (res != str)
-    str->copy(res->ptr(), res->length(), res->charset());
+    str->copy_or_move(res->ptr(), res->length(), res->charset());
 
   for (uint i= 1 ; i < arg_count ; i++)
   {

sql/mysqld.cc

@@ -2992,10 +2992,6 @@ static bool cache_thread(THD *thd)
       _db_pop_();
 #endif
 
-    /* Clear warnings. */
-    if (!thd->get_stmt_da()->is_warning_info_empty())
-      thd->get_stmt_da()->clear_warning_info(thd->query_id);
-
     set_timespec(abstime, THREAD_CACHE_TIMEOUT);
     while (!abort_loop && ! wake_thread && ! kill_cached_threads)
     {

sql/sql_acl.cc

@@ -12285,7 +12285,7 @@ static bool send_plugin_request_packet(MPVIO_EXT *mpvio,
   const char *client_auth_plugin=
     ((st_mysql_auth *) (plugin_decl(mpvio->plugin)->info))->client_auth_plugin;
 
-  DBUG_EXECUTE_IF("auth_disconnect", { vio_close(net->vio); DBUG_RETURN(1); });
+  DBUG_EXECUTE_IF("auth_disconnect", { DBUG_RETURN(1); });
   DBUG_ASSERT(client_auth_plugin);
 
   /*

sql/sql_class.cc

@@ -1363,6 +1363,11 @@ void THD::change_user(void)
   cleanup_done= 0;
   reset_killed();
   thd_clear_errors(this);
+
+  /* Clear warnings. */
+  if (!get_stmt_da()->is_warning_info_empty())
+    get_stmt_da()->clear_warning_info(0);
+
   init();
   stmt_map.reset();
   my_hash_init(&user_vars, system_charset_info, USER_VARS_HASH_SIZE, 0, 0,

sql/sql_prepare.cc

@@ -3212,7 +3212,7 @@ static void mysql_stmt_execute_common(THD *thd,
   sp_cache_enforce_limit(thd->sp_package_body_cache, stored_program_cache_size);
 
   /* Close connection socket; for use with client testing (Bug#43560). */
-  DBUG_EXECUTE_IF("close_conn_after_stmt_execute", vio_close(thd->net.vio););
+  DBUG_EXECUTE_IF("close_conn_after_stmt_execute", vio_shutdown(thd->net.vio,SHUT_RD););
 
   DBUG_VOID_RETURN;
 }

sql/sql_string.cc

@@ -238,9 +238,9 @@ bool String::copy(const String &str)
 
 bool String::copy(const char *str,size_t arg_length, CHARSET_INFO *cs)
 {
+  DBUG_ASSERT(arg_length < UINT_MAX32);
   if (alloc(arg_length))
     return TRUE;
-  DBUG_ASSERT(arg_length < UINT_MAX32);
   if (Ptr == str && arg_length == uint32(str_length))
   {
     /*
@@ -257,6 +257,24 @@ bool String::copy(const char *str,size_t arg_length, CHARSET_INFO *cs)
   return FALSE;
 }
 
+/*
+  Copy string, where strings may overlap.
+  Same as String::copy, but use memmove instead of memcpy to avoid warnings
+  from valgrind
+*/
+
+bool String::copy_or_move(const char *str,size_t arg_length, CHARSET_INFO *cs)
+{
+  DBUG_ASSERT(arg_length < UINT_MAX32);
+  if (alloc(arg_length))
+    return TRUE;
+  if ((str_length=uint32(arg_length)))
+    memmove(Ptr,str,arg_length);
+  Ptr[arg_length]=0;
+  str_charset=cs;
+  return FALSE;
+}
+
 
 /*
   Checks that the source string can be just copied to the destination string
@@ -390,8 +408,9 @@ bool String::set_or_copy_aligned(const char *str, size_t arg_length,
   /* How many bytes are in incomplete character */
   size_t offset= (arg_length % cs->mbminlen); 
   
-  if (!offset) /* All characters are complete, just copy */
+  if (!offset)
   {
+    /* All characters are complete, just use given string */
     set(str, arg_length, cs);
     return FALSE;
   }

sql/sql_string.h

@@ -437,6 +437,7 @@ public:
   bool copy();					// Alloc string if not alloced
   bool copy(const String &s);			// Allocate new string
   bool copy(const char *s,size_t arg_length, CHARSET_INFO *cs);	// Allocate new string
+  bool copy_or_move(const char *s,size_t arg_length, CHARSET_INFO *cs);
   static bool needs_conversion(size_t arg_length,
   			       CHARSET_INFO *cs_from, CHARSET_INFO *cs_to,
 			       uint32 *offset);

storage/connect/CMakeLists.txt

@@ -128,7 +128,6 @@ IF(WIN32)
   OPTION(CONNECT_WITH_MSXML "Compile CONNECT storage engine with MSXML support" ON)
   IF(CONNECT_WITH_MSXML)
     add_definitions(-DMSX6 -DDOMDOC_SUPPORT)
-    message(STATUS "MSXML library version: msxml6")
     SET(MSXML_FOUND 1)
     SET(CONNECT_SOURCES ${CONNECT_SOURCES} domdoc.cpp domdoc.h)
   ENDIF(CONNECT_WITH_MSXML)

storage/innobase/dict/dict0crea.cc

@@ -2019,86 +2019,6 @@ dict_create_add_foreign_to_dictionary(
 	DBUG_RETURN(error);
 }
 
-/** Check whether a column is in an index by the column name
-@param[in]	col_name	column name for the column to be checked
-@param[in]	index		the index to be searched
-@return	true if this column is in the index, otherwise, false */
-static
-bool
-dict_index_has_col_by_name(
-/*=======================*/
-	const char*		col_name,
-	const dict_index_t*	index)
-{
-	for (ulint i = 0; i < index->n_fields; i++) {
-		dict_field_t*   field = dict_index_get_nth_field(index, i);
-
-		if (strcmp(field->name, col_name) == 0) {
-			return(true);
-		}
-	}
-	return(false);
-}
-
-/** Check whether the foreign constraint could be on a column that is
-part of a virtual index (index contains virtual column) in the table
-@param[in]	fk_col_name	FK column name to be checked
-@param[in]	table		the table
-@return	true if this column is indexed with other virtual columns */
-bool
-dict_foreign_has_col_in_v_index(
-	const char*		fk_col_name,
-	const dict_table_t*	table)
-{
-	/* virtual column can't be Primary Key, so start with secondary index */
-	for (dict_index_t* index = dict_table_get_next_index(
-		     dict_table_get_first_index(table));
-	     index;
-	     index = dict_table_get_next_index(index)) {
-
-		if (dict_index_has_virtual(index)) {
-			if (dict_index_has_col_by_name(fk_col_name, index)) {
-				return(true);
-			}
-		}
-	}
-
-	return(false);
-}
-
-
-/** Check whether the foreign constraint could be on a column that is
-a base column of some indexed virtual columns.
-@param[in]	col_name	column name for the column to be checked
-@param[in]	table		the table
-@return	true if this column is a base column, otherwise, false */
-bool
-dict_foreign_has_col_as_base_col(
-	const char*		col_name,
-	const dict_table_t*	table)
-{
-	/* Loop through each virtual column and check if its base column has
-	the same name as the column name being checked */
-	for (ulint i = 0; i < table->n_v_cols; i++) {
-		dict_v_col_t*	v_col = dict_table_get_nth_v_col(table, i);
-
-		/* Only check if the virtual column is indexed */
-		if (!v_col->m_col.ord_part) {
-			continue;
-		}
-
-		for (ulint j = 0; j < v_col->num_base; j++) {
-			if (strcmp(col_name, dict_table_get_col_name(
-					   table,
-					   v_col->base_col[j]->ind)) == 0) {
-				return(true);
-			}
-		}
-	}
-
-	return(false);
-}
-
 /** Check if a foreign constraint is on the given column name.
 @param[in]	col_name	column name to be searched for fk constraint
 @param[in]	table		table to which foreign key constraint belongs
@@ -2173,43 +2093,6 @@ dict_foreigns_has_s_base_col(
 	return(false);
 }
 
-/** Check if a column is in foreign constraint with CASCADE properties or
-SET NULL
-@param[in]	table		table
-@param[in]	fk_col_name	name for the column to be checked
-@return true if the column is in foreign constraint, otherwise, false */
-bool
-dict_foreigns_has_this_col(
-	const dict_table_t*	table,
-	const char*		col_name)
-{
-	dict_foreign_t*		foreign;
-	const dict_foreign_set*	local_fk_set = &table->foreign_set;
-
-	for (dict_foreign_set::const_iterator it = local_fk_set->begin();
-	     it != local_fk_set->end();
-	     ++it) {
-		foreign = *it;
-		ut_ad(foreign->id != NULL);
-		ulint	type = foreign->type;
-
-		type &= ~(DICT_FOREIGN_ON_DELETE_NO_ACTION
-			  | DICT_FOREIGN_ON_UPDATE_NO_ACTION);
-
-		if (type == 0) {
-			continue;
-		}
-
-		for (ulint i = 0; i < foreign->n_fields; i++) {
-			if (strcmp(foreign->foreign_col_names[i],
-				   col_name) == 0) {
-				return(true);
-			}
-		}
-	}
-	return(false);
-}
-
 /** Adds the given set of foreign key objects to the dictionary tables
 in the database. This function does not modify the dictionary cache. The
 caller must ensure that all foreign key objects contain a valid constraint

storage/innobase/handler/ha_innodb.cc

@@ -20491,13 +20491,12 @@ static TABLE* innodb_find_table_for_vc(THD* thd, dict_table_t* table)
 }
 
 /** Get the computed value by supplying the base column values.
-@param[in,out]	table	table whose virtual column template to be built */
+@param[in,out]	table		table whose virtual column
-void
+				template to be built */
-innobase_init_vc_templ(
+TABLE* innobase_init_vc_templ(dict_table_t* table)
-	dict_table_t*	table)
 {
 	if (table->vc_templ != NULL) {
-		return;
+		return NULL;
 	}
 
 	table->vc_templ = UT_NEW_NOKEY(dict_vcol_templ_t());
@@ -20506,12 +20505,13 @@ innobase_init_vc_templ(
 
 	ut_ad(mysql_table);
 	if (!mysql_table) {
-		return;
+		return NULL;
 	}
 
 	mutex_enter(&dict_sys->mutex);
 	innobase_build_v_templ(mysql_table, table, table->vc_templ, NULL, true);
 	mutex_exit(&dict_sys->mutex);
+	return mysql_table;
 }
 
 /** Change dbname and table name in table->vc_templ.

storage/innobase/include/row0mysql.h

@@ -944,10 +944,9 @@ innobase_get_computed_value(
 	dict_foreign_t*		foreign);
 
 /** Get the computed value by supplying the base column values.
-@param[in,out]	table	the table whose virtual column template to be built */
+@param[in,out]	table		the table whose virtual column
-void
+				template to be built */
-innobase_init_vc_templ(
+TABLE* innobase_init_vc_templ(dict_table_t* table);
-	dict_table_t*	table);
 
 /** Change dbname and table name in table->vc_templ.
 @param[in,out]	table	the table whose virtual column template

storage/innobase/log/log0recv.cc

@@ -3504,6 +3504,8 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 		then there is a possiblity that hash table will not contain
 		all space ids redo logs. Rescan the remaining unstored
 		redo logs for the validation of missing tablespace. */
+		ut_ad(rescan || !missing_tablespace);
+
 		while (missing_tablespace) {
 			DBUG_PRINT("ib_log", ("Rescan of redo log to validate "
 					      "the missing tablespace. Scan "
@@ -3527,6 +3529,8 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 				log_mutex_exit();
 				return err;
 			}
+
+			rescan = true;
 		}
 
 		if (srv_operation == SRV_OPERATION_NORMAL) {

storage/innobase/row/row0purge.cc

@@ -375,6 +375,13 @@ retry_purge_sec:
 
 	ut_ad(mtr.has_committed());
 
+	/* If the virtual column info is not used then reset the virtual column
+	info. */
+	if (node->vcol_info.is_requested()
+	    && !node->vcol_info.is_used()) {
+		node->vcol_info.reset();
+	}
+
 	if (store_cur && !row_purge_restore_vsec_cur(
 		    node, index, sec_pcur, sec_mtr, is_tree)) {
 		return false;
@@ -1100,8 +1107,10 @@ try_again:
 			goto try_again;
 		}
 
-		/* Initialize the template for the table */
+		node->vcol_info.set_requested();
-		innobase_init_vc_templ(node->table);
+		node->vcol_info.set_used();
+		node->vcol_info.set_table(innobase_init_vc_templ(node->table));
+		node->vcol_info.set_used();
 	}
 
 	clust_index = dict_table_get_first_index(node->table);

storage/innobase/sync/sync0rw.cc

@@ -487,14 +487,6 @@ rw_lock_x_lock_wait_func(
 		lock->count_os_wait += static_cast<uint32_t>(count_os_wait);
 		rw_lock_stats.rw_x_os_wait_count.add(count_os_wait);
 	}
-
-	rw_lock_stats.rw_x_spin_round_count.add(n_spins);
-
-	if (count_os_wait > 0) {
-		lock->count_os_wait +=
-			static_cast<uint32_t>(count_os_wait);
-		rw_lock_stats.rw_x_os_wait_count.add(count_os_wait);
-	}
 }
 
 #ifdef UNIV_DEBUG

storage/maria/ma_key_recover.c

@@ -933,8 +933,6 @@ err:
      @retval 1      Error
 */
 
-long my_counter= 0;
-
 uint _ma_apply_redo_index(MARIA_HA *info,
                           LSN lsn, const uchar *header, uint head_length)
 {

storage/mroonga/CMakeLists.txt

@@ -65,7 +65,7 @@ set(MRN_BUNDLED_GROONGA_DIR
 if(EXISTS "${MRN_BUNDLED_GROONGA_DIR}")
   set(MRN_GROONGA_BUNDLED TRUE)
   if(MSVC)
-    message(STATUS "Bundled Mroonga does not support MSVC yet")
+    # Bundled Mroonga does not support MSVC yet
     return()
   endif()
 else()

storage/tokudb/CMakeLists.txt

@@ -1,6 +1,9 @@
 SET(TOKUDB_VERSION 5.6.41-84.1)
 # PerconaFT only supports x86-64 and cmake-2.8.9+
-IF(CMAKE_VERSION VERSION_LESS "2.8.9")
+IF(WIN32)
+  # tokudb never worked there
+  RETURN()
+ELSEIF(CMAKE_VERSION VERSION_LESS "2.8.9")
   MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB")
 ELSEIF(NOT HAVE_DLOPEN)
   MESSAGE(STATUS "dlopen is required by TokuDB")

tests/mysql_client_test.c

@@ -16425,6 +16425,7 @@ static void test_change_user()
   const char *db= "mysqltest_user_test_database";
   int rc;
   MYSQL*       conn;
+  MYSQL_RES* res;
   DBUG_ENTER("test_change_user");
   myheader("test_change_user");
 
@@ -16561,6 +16562,20 @@ static void test_change_user()
   rc= mysql_change_user(conn, user_pw, pw, "");
   myquery(rc);
 
+  /* MDEV-14581 : Check that there are no warnings after change user.*/
+  rc = mysql_query(conn,"SIGNAL SQLSTATE '01000'");
+  myquery(rc);
+
+  rc = mysql_change_user(conn, user_pw, pw, "");
+  myquery(rc);
+
+  rc = mysql_query(conn, "SHOW WARNINGS");
+  myquery(rc);
+  res = mysql_store_result(conn);
+  rc = my_process_result_set(res);
+  DIE_UNLESS(rc == 0);
+  mysql_free_result(res);
+
   rc= mysql_change_user(conn, user_no_pw, pw, db);
   DIE_UNLESS(rc);
   if (! opt_silent)