Merge 10.3 into 10.4

extra/mariabackup/xtrabackup.cc

@@ -4672,12 +4672,11 @@ fail_before_log_copying_thread_start:
 	log_file_op = NULL;
 	pthread_mutex_destroy(&backup_mutex);
 	pthread_cond_destroy(&scanned_lsn_cond);
-	if (opt_log_innodb_page_corruption && !corrupted_pages.empty()) {
+	if (!corrupted_pages.empty()) {
+		ut_ad(opt_log_innodb_page_corruption);
 		msg("Error: corrupted innodb pages are found and logged to "
 			MB_CORRUPTED_PAGES_FILE " file");
-		return false;
 	}
-	else
 	return(true);
 }
 

mysql-test/main/win.result

@@ -3866,6 +3866,32 @@ NULL
 DROP VIEW v1;
 DROP TABLE t1,t2;
 #
+# MDEV-25032 Window functions without column references get removed from ORDER BY
+#
+create table t1 (id int, score double);
+insert into t1 values
+(1, 5),
+(1, 6),
+(1, 6),
+(1, 6),
+(1, 7),
+(1, 8.1),
+(1, 9),
+(1, 10);
+select id, row_number() over () rn
+from t1
+order by rn desc;
+id	rn
+1	8
+1	7
+1	6
+1	5
+1	4
+1	3
+1	2
+1	1
+drop table t1;
+#
 # End of 10.2 tests
 #
 #

mysql-test/main/win.test

@@ -2522,6 +2522,26 @@ SELECT NTH_VALUE(i1, i1) OVER (PARTITION BY i1) FROM v1;
 DROP VIEW v1;
 DROP TABLE t1,t2;
 
+--echo #
+--echo # MDEV-25032 Window functions without column references get removed from ORDER BY
+--echo #
+
+create table t1 (id int, score double);
+insert into t1 values
+(1, 5),
+(1, 6),
+(1, 6),
+(1, 6),
+(1, 7),
+(1, 8.1),
+(1, 9),
+(1, 10);
+select id, row_number() over () rn
+from t1
+order by rn desc;
+
+drop table t1;
+
 --echo #
 --echo # End of 10.2 tests
 --echo #

mysql-test/suite/encryption/r/tempfiles_encrypted.result

@@ -3872,6 +3872,32 @@ NULL
 DROP VIEW v1;
 DROP TABLE t1,t2;
 #
+# MDEV-25032 Window functions without column references get removed from ORDER BY
+#
+create table t1 (id int, score double);
+insert into t1 values
+(1, 5),
+(1, 6),
+(1, 6),
+(1, 6),
+(1, 7),
+(1, 8.1),
+(1, 9),
+(1, 10);
+select id, row_number() over () rn
+from t1
+order by rn desc;
+id	rn
+1	8
+1	7
+1	6
+1	5
+1	4
+1	3
+1	2
+1	1
+drop table t1;
+#
 # End of 10.2 tests
 #
 #

mysql-test/suite/innodb/r/innodb.result

@@ -3319,3 +3319,20 @@ c1	c2
 9	3
 DROP TABLE t1;
 DROP TABLE t2;
+#
+# MDEV-24748 Extern field check missing
+#	in btr_index_rec_validate()
+#
+CREATE TABLE t1 (pk INT, c1 char(255),
+c2 char(255), c3 char(255), c4 char(255),
+c5 char(255), c6 char(255), c7 char(255),
+c8 char(255), primary key (pk)
+) CHARACTER SET utf32 ENGINE=InnoDB;
+INSERT INTO t1 VALUES
+(1, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'),
+(2, 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p');
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+ALTER TABLE t1 FORCE;
+DROP TABLE t1;

mysql-test/suite/innodb/t/innodb.test

@@ -2592,3 +2592,21 @@ SELECT * FROM t2;
 
 DROP TABLE t1;
 DROP TABLE t2;
+
+--echo #
+--echo # MDEV-24748 Extern field check missing
+--echo #	in btr_index_rec_validate()
+--echo #
+CREATE TABLE t1 (pk INT, c1 char(255),
+c2 char(255), c3 char(255), c4 char(255),
+c5 char(255), c6 char(255), c7 char(255),
+c8 char(255), primary key (pk)
+) CHARACTER SET utf32 ENGINE=InnoDB;
+
+INSERT INTO t1 VALUES
+  (1, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'),
+  (2, 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p');
+CHECK TABLE t1;
+ALTER TABLE t1 FORCE;
+# Cleanup
+DROP TABLE t1;

mysql-test/suite/mariabackup/log_page_corruption.result

@@ -26,8 +26,9 @@ INSERT INTO t7_corrupted_to_alter VALUES (3), (4), (5), (6), (7), (8), (9);
 # Backup must  fail due to page corruption
 FOUND 1 /Database page corruption detected.*/ in backup.log
 # "innodb_corrupted_pages" file must not exist
-# Backup must fail, but "innodb_corrupted_pages" file must be created due to --log-innodb-page-corruption option
+# Backup must not fail, but "innodb_corrupted_pages" file must be created due to --log-innodb-page-corruption option
 FOUND 1 /Database page corruption detected.*/ in backup.log
+FOUND 1 /completed OK!/ in backup.log
 --- "innodb_corrupted_pages" file content: ---
 test/t1_corrupted
 6 8 9
@@ -44,7 +45,7 @@ INSERT INTO t1_inc_corrupted VALUES (3), (4), (5), (6), (7), (8), (9);
 INSERT INTO t2_inc_corrupted VALUES (3), (4), (5), (6), (7), (8), (9);
 INSERT INTO t3_inc VALUES (3), (4), (5), (6), (7), (8), (9);
 # restart
-# Backup must fail, but "innodb_corrupted_pages" file must be created due to --log-innodb-page-corruption option
+# Backup must not fail, but "innodb_corrupted_pages" file must be created due to --log-innodb-page-corruption option
 --- "innodb_corrupted_pages" file content: ---
 test/t1_corrupted
 6 8 9

mysql-test/suite/mariabackup/log_page_corruption.test

@@ -80,15 +80,19 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=
 --let after_copy_test_t7_corrupted_to_alter=ALTER TABLE test.t7_corrupted_to_alter ADD COLUMN (d INT)
 --let add_corrupted_page_for_test_t7_corrupted_to_alter=3
 
---echo # Backup must fail, but "innodb_corrupted_pages" file must be created due to --log-innodb-page-corruption option
+--echo # Backup must not fail, but "innodb_corrupted_pages" file must be created due to --log-innodb-page-corruption option
 --disable_result_log
---error 1
 --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --log-innodb-page-corruption --target-dir=$targetdir --dbug=+d,mariabackup_events,mariabackup_inject_code  > $backuplog
 --enable_result_log
 
 --let SEARCH_PATTERN=Database page corruption detected.*
 --let SEARCH_FILE=$backuplog
 --source include/search_pattern_in_file.inc
+
+--let SEARCH_PATTERN=completed OK!
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+
 --echo --- "innodb_corrupted_pages" file content: ---
 perl;
 do "$ENV{MTR_SUITE_DIR}/include/corrupt-page.pl";
@@ -145,9 +149,8 @@ EOF
 --let after_copy_test_t7_inc_corrupted_to_alter=ALTER TABLE test.t7_inc_corrupted_to_alter ADD COLUMN (d INT)
 --let add_corrupted_page_for_test_t7_inc_corrupted_to_alter=3
 
---echo # Backup must fail, but "innodb_corrupted_pages" file must be created due to --log-innodb-page-corruption option
+--echo # Backup must not fail, but "innodb_corrupted_pages" file must be created due to --log-innodb-page-corruption option
 --disable_result_log
---error 1
 --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --log-innodb-page-corruption --target-dir=$incdir --incremental-basedir=$targetdir --dbug=+d,mariabackup_events,mariabackup_inject_code  > $backuplog
 --disable_result_log
 
@@ -161,6 +164,9 @@ EOF
 --let SEARCH_PATTERN=Database page corruption detected.*
 --let SEARCH_FILE=$backuplog
 --source include/search_pattern_in_file.inc
+--let SEARCH_PATTERN=completed OK!
+--source include/search_pattern_in_file.inc
+
 --let corrupted_pages_file = $incdir/innodb_corrupted_pages
 --echo --- "innodb_corrupted_pages" file content: ---
 perl;
@@ -260,7 +266,6 @@ EOF
 
 --echo # Full backup with --log-innodb-page-corruption
 --disable_result_log
---error 1
 --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --log-innodb-page-corruption --target-dir=$targetdir
 --enable_result_log
 --let corrupted_pages_file = $targetdir/innodb_corrupted_pages
@@ -288,7 +293,6 @@ EOF
 
 --echo # Incremental backup --log-innodb-page-corruption
 --disable_result_log
---error 1
 --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --log-innodb-page-corruption --target-dir=$incdir --incremental-basedir=$targetdir --dbug=+d,mariabackup_events,mariabackup_inject_code  > $backuplog
 --disable_result_log
 --let corrupted_pages_file = $incdir/innodb_corrupted_pages

mysql-test/suite/unit/suite.pm

@@ -48,10 +48,10 @@ sub start_test {
   my ($command, %tests, $prefix);
   for (@ctest_list) {
     chomp;
-    if (/^\d+: Test command: +([^ \t]+)/) {
+    if (/^\d+: Test command: +([^ \t]+.*)/) {
       $command= $1;
       $prefix= /libmariadb/ ? 'conc_' : '';
-    } elsif (/^ +Test +#\d+: ([^ \t]+)/) {
+    } elsif (/^ +Test +#\d+: ([^ \t]+.*)/) {
       if ($command ne "NOT_AVAILABLE" && $command ne "/bin/sh") {
         $tests{$prefix.$1}=$command;
       }

sql/sql_select.cc

@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2016, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2020, MariaDB Corporation.
+   Copyright (c) 2009, 2021, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -14140,6 +14140,7 @@ remove_const(JOIN *join,ORDER *first_order, COND *cond,
   {
     table_map order_tables=order->item[0]->used_tables();
     if (order->item[0]->with_sum_func() ||
+        order->item[0]->with_window_func ||
         /*
           If the outer table of an outer join is const (either by itself or
           after applying WHERE condition), grouping on a field from such a

storage/innobase/btr/btr0btr.cc

@@ -2,7 +2,7 @@
 
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2014, 2020, MariaDB Corporation.
+Copyright (c) 2014, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -4666,6 +4666,16 @@ n_field_mismatch:
 		} else {
 			fixed_size = dict_col_get_fixed_size(
 				field->col, page_is_comp(page));
+			if (rec_offs_nth_extern(offsets, i)) {
+				const byte* data = rec_get_nth_field(
+					rec, offsets, i, &len);
+				len -= BTR_EXTERN_FIELD_REF_SIZE;
+				ulint extern_len = mach_read_from_4(
+					data + len + BTR_EXTERN_LEN + 4);
+				if (fixed_size == extern_len) {
+					continue;
+				}
+			}
 		}
 
 		/* Note that if fixed_size != 0, it equals the

storage/innobase/fil/fil0fil.cc

@@ -887,15 +887,14 @@ fil_space_extend_must_retry(
 	}
 }
 
-/*******************************************************************//**
+/** Reserves the fil_system.mutex and tries to make sure we can open at least one
-Reserves the fil_system.mutex and tries to make sure we can open at least one
 file while holding it. This should be called before calling
-fil_node_prepare_for_io(), because that function may need to open a file. */
+fil_node_prepare_for_io(), because that function may need to open a file.
+@param[in]	space_id	tablespace id
+@return whether the tablespace is usable for io */
 static
-void
+bool
-fil_mutex_enter_and_prepare_for_io(
+fil_mutex_enter_and_prepare_for_io(ulint space_id)
-/*===============================*/
-	ulint	space_id)	/*!< in: space id */
 {
 	for (ulint count = 0;;) {
 		mutex_enter(&fil_system.mutex);
@@ -908,7 +907,7 @@ fil_mutex_enter_and_prepare_for_io(
 		fil_space_t*	space = fil_space_get_by_id(space_id);
 
 		if (space == NULL) {
-			break;
+			return false;
 		}
 
 		fil_node_t*	node = UT_LIST_GET_LAST(space->chain);
@@ -923,6 +922,10 @@ fil_mutex_enter_and_prepare_for_io(
 			the insert buffer. The insert buffer is in
 			tablespace 0, and we cannot end up waiting in
 			this function. */
+		} else if (space->is_stopping() && !space->is_being_truncated) {
+			/* If the tablespace is being deleted then InnoDB
+			shouldn't prepare the tablespace for i/o */
+			return false;
 		} else if (!node || node->is_open()) {
 			/* If the file is already open, no need to do
 			anything; if the space does not exist, we handle the
@@ -994,6 +997,8 @@ fil_mutex_enter_and_prepare_for_io(
 
 		break;
 	}
+
+	return true;
 }
 
 /** Try to extend a tablespace if it is smaller than the specified size.
@@ -1010,7 +1015,10 @@ fil_space_extend(
 	bool	success;
 
 	do {
-		fil_mutex_enter_and_prepare_for_io(space->id);
+		if (!fil_mutex_enter_and_prepare_for_io(space->id)) {
+			success = false;
+			break;
+		}
 	} while (fil_space_extend_must_retry(
 			 space, UT_LIST_GET_LAST(space->chain), size,
 			 &success));
@@ -1365,7 +1373,9 @@ fil_space_t* fil_system_t::read_page0(ulint id)
 
 	/* It is possible that the tablespace is dropped while we are
 	not holding the mutex. */
-	fil_mutex_enter_and_prepare_for_io(id);
+	if (!fil_mutex_enter_and_prepare_for_io(id)) {
+		return NULL;
+	}
 
 	fil_space_t* space = fil_space_get_by_id(id);
 

storage/innobase/handler/ha_innodb.cc

@@ -13334,17 +13334,10 @@ innobase_drop_database(
 @param[in,out]	trx	InnoDB data dictionary transaction
 @param[in]	from	old table name
 @param[in]	to	new table name
-@param[in]	commit	whether to commit trx
+@param[in]	commit	whether to commit trx (and to enforce FOREIGN KEY)
-@param[in]	use_fk	whether to parse and enforce FOREIGN KEY constraints
 @return DB_SUCCESS or error code */
-inline
+inline dberr_t innobase_rename_table(trx_t *trx, const char *from,
-dberr_t
+                                     const char *to, bool commit)
-innobase_rename_table(
-	trx_t*		trx,
-	const char*	from,
-	const char*	to,
-	bool		commit,
-	bool		use_fk)
 {
 	dberr_t	error;
 	char	norm_to[FN_REFLEN];
@@ -13377,6 +13370,9 @@ innobase_rename_table(
 	Convert lock_wait_timeout unit from second to 250 milliseconds */
 	long int lock_wait_timeout = thd_lock_wait_timeout(trx->mysql_thd) * 4;
 	if (table != NULL) {
+		if (commit) {
+			dict_stats_wait_bg_to_stop_using_table(table, trx);
+		}
 		for (dict_index_t* index = dict_table_get_first_index(table);
 		     index != NULL;
 		     index = dict_table_get_next_index(index)) {
@@ -13390,8 +13386,10 @@ innobase_rename_table(
 				}
 			}
 		}
+		if (!commit) {
 			dict_table_close(table, TRUE, FALSE);
 		}
+	}
 
 	/* FTS sync is in progress. We shall timeout this operation */
 	if (lock_wait_timeout < 0) {
@@ -13400,7 +13398,7 @@ innobase_rename_table(
 	}
 
 	error = row_rename_table_for_mysql(norm_from, norm_to, trx, commit,
-					   use_fk);
+					   commit);
 
 	if (error != DB_SUCCESS) {
 		if (error == DB_TABLE_NOT_FOUND
@@ -13452,6 +13450,10 @@ innobase_rename_table(
 
 func_exit:
 	if (commit) {
+		if (table) {
+			table->stats_bg_flag &= ~BG_STAT_SHOULD_QUIT;
+			dict_table_close(table, TRUE, FALSE);
+		}
 		row_mysql_unlock_data_dictionary(trx);
 	}
 
@@ -13506,9 +13508,11 @@ int ha_innobase::truncate()
 	++trx->will_lock;
 	trx_set_dict_operation(trx, TRX_DICT_OP_TABLE);
 	row_mysql_lock_data_dictionary(trx);
+	dict_stats_wait_bg_to_stop_using_table(ib_table, trx);
+
 	int err = convert_error_code_to_mysql(
 		innobase_rename_table(trx, ib_table->name.m_name, temp_name,
-				      false, false),
+				      false),
 		ib_table->flags, m_user_thd);
 	if (err) {
 		trx_rollback_for_mysql(trx);
@@ -13591,7 +13595,7 @@ ha_innobase::rename_table(
 	++trx->will_lock;
 	trx_set_dict_operation(trx, TRX_DICT_OP_INDEX);
 
-	dberr_t	error = innobase_rename_table(trx, from, to, true, true);
+	dberr_t	error = innobase_rename_table(trx, from, to, true);
 
 	DEBUG_SYNC(thd, "after_innobase_rename_table");
 

storage/innobase/include/row0mysql.h

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2019, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -804,12 +804,6 @@ struct row_prebuilt_t {
 					search key values from MySQL format
 					to InnoDB format.*/
 	uint		srch_key_val_len; /*!< Size of search key */
-	/** Disable prefetch. */
-	bool		m_no_prefetch;
-
-	/** Return materialized key for secondary index scan */
-	bool		m_read_virtual_key;
-
 	/** The MySQL table object */
 	TABLE*		m_mysql_table;
 

storage/innobase/row/row0mysql.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2000, 2018, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2015, 2020, MariaDB Corporation.
+Copyright (c) 2015, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -964,9 +964,6 @@ row_create_prebuilt(
 	prebuilt->fts_doc_id_in_read_set = 0;
 	prebuilt->blob_heap = NULL;
 
-	prebuilt->m_no_prefetch = false;
-	prebuilt->m_read_virtual_key = false;
-
 	DBUG_RETURN(prebuilt);
 }
 

storage/innobase/row/row0sel.cc

@@ -2,7 +2,7 @@
 
 Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2008, Google Inc.
-Copyright (c) 2015, 2020, MariaDB Corporation.
+Copyright (c) 2015, 2021, MariaDB Corporation.
 
 Portions of this file contain modifications contributed and copyrighted by
 Google, Inc. Those modifications are gratefully acknowledged and are described
@@ -3037,8 +3037,7 @@ static bool row_sel_store_mysql_rec(
 			search or virtual key read is not requested. */
 			if (!rec_clust
 			    || !prebuilt->index->has_virtual()
-			    || (!prebuilt->read_just_key
+			    || !prebuilt->read_just_key) {
-				&& !prebuilt->m_read_virtual_key)) {
 				/* Initialize the NULL bit. */
 				if (templ->mysql_null_bit_mask) {
 					mysql_rec[templ->mysql_null_byte_offset]
@@ -3056,23 +3055,8 @@ static bool row_sel_store_mysql_rec(
 			const dfield_t* dfield = dtuple_get_nth_v_field(
 				vrow, col->v_pos);
 
-			/* If this is a partitioned table, it might request
-			InnoDB to fill out virtual column data for serach
-			index key values while other non key columns are also
-			getting selected. The non-key virtual columns may
-			not be materialized and we should skip them. */
 			if (dfield_get_type(dfield)->mtype == DATA_MISSING) {
-#ifdef UNIV_DEBUG
+				ut_ad("no ha_innopart in MariaDB" == 0);
-				ulint prefix;
-#endif /* UNIV_DEBUG */
-				ut_ad(prebuilt->m_read_virtual_key);
-
-				/* If it is part of index key the data should
-				have been materialized. */
-				ut_ad(dict_index_get_nth_col_or_prefix_pos(
-					prebuilt->index, col->v_pos, false,
-					true, &prefix) == ULINT_UNDEFINED);
-
 				continue;
 			}
 
@@ -4289,8 +4273,7 @@ row_search_mvcc(
 	index key, if this is covered index scan or virtual key read is
 	requested. */
 	bool    need_vrow = dict_index_has_virtual(prebuilt->index)
-		&& (prebuilt->read_just_key
+		&& prebuilt->read_just_key;
-		    || prebuilt->m_read_virtual_key);
 
 	/* Reset the new record lock info if srv_locks_unsafe_for_binlog
 	is set or session is using a READ COMMITTED isolation level. Then
@@ -5435,7 +5418,6 @@ use_covering_index:
 	if ((match_mode == ROW_SEL_EXACT
 	     || prebuilt->n_rows_fetched >= MYSQL_FETCH_CACHE_THRESHOLD)
 	    && prebuilt->select_lock_type == LOCK_NONE
-	    && !prebuilt->m_no_prefetch
 	    && !prebuilt->templ_contains_blob
 	    && !prebuilt->clust_index_was_generated
 	    && !prebuilt->used_in_HANDLER