From 058554027f6696775ca1b289688956606f59ce7d Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Thu, 5 Jul 2018 00:06:39 -0700
Subject: [PATCH 1/5] MDEV-16629 "Table Does Not Exist" Error from Recursive
 CTE Query Inside Function

When processing a query containing with clauses a call of the function
check_dependencies_in_with_clauses() before opening tables used in the
query is necessary if with clauses include specifications of recursive
CTEs.
This call was missing if such a query belonged to a stored function.
This caused misbehavior of the server: it could report a fake error
as in the test case for MDEV-16629 or the executed query could hang
as in the test cases for MDEV-16661 and MDEV-15151.
---
 mysql-test/r/cte_recursive.result | 58 +++++++++++++++++++++++++
 mysql-test/t/cte_recursive.test   | 72 +++++++++++++++++++++++++++++++
 sql/sp_head.cc                    |  4 +-
 3 files changed, 133 insertions(+), 1 deletion(-)

diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result
index 2e93e9f6ff8..856f00abe59 100644
--- a/mysql-test/r/cte_recursive.result
+++ b/mysql-test/r/cte_recursive.result
@@ -3263,3 +3263,61 @@ select 3, 0*(@d:=@d+1) from qn where @d<1
 select * from qn;
 ERROR 42000: This version of MariaDB doesn't yet support 'mix of ALL and DISTINCT UNION operations in recursive CTE spec'
 drop table t1;
+#
+# MDEV-16629: function with recursive CTE using a base table
+#
+CREATE TABLE t1 (id int);
+INSERT INTO t1 VALUES (0), (1),(2);
+WITH recursive cte AS
+(SELECT id FROM t1 UNION SELECT 3 FROM cte)
+SELECT count(id) FROM cte;
+count(id)
+4
+CREATE OR REPLACE FUNCTION func() RETURNS int
+RETURN
+(
+WITH recursive cte AS
+(SELECT id FROM t1 UNION SELECT 3 FROM cte)
+SELECT count(id) FROM cte
+);
+SELECT func();
+func()
+4
+DROP FUNCTION func;
+DROP TABLE t1;
+#
+# MDEV-16661: function with recursive CTE using no base tables
+#             (fixed by the patch for MDEV-16629)
+#
+CREATE OR REPLACE FUNCTION func() RETURNS int
+RETURN
+(
+WITH RECURSIVE cte AS
+(SELECT 1 as id UNION SELECT * FROM cte)
+SELECT count(id) FROM cte
+);
+SELECT func();
+func()
+1
+DROP FUNCTION func;
+#
+# MDEV-15151: function with recursive CTE using no base tables
+#             (duplicate of  MDEV-16661)
+#
+connection default;
+CREATE TABLE t1 (id int  KEY);
+INSERT INTO t1 VALUES (0), (1),(2);
+CREATE OR REPLACE FUNCTION func() RETURNS int
+RETURN
+(
+WITH recursive cte AS
+(SELECT 1 a UNION SELECT cte.* FROM cte natural join t1)
+SELECT * FROM cte  limit 1
+);
+SELECT func();;
+connect  con1,localhost,root,,;
+KILL QUERY 4;
+DROP FUNCTION func;
+DROP TABLE t1;
+disconnect con1;
+connection default;
diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test
index 32a82c494e0..504b2c64442 100644
--- a/mysql-test/t/cte_recursive.test
+++ b/mysql-test/t/cte_recursive.test
@@ -2282,3 +2282,75 @@ select 3, 0*(@d:=@d+1) from qn where @d<1
 select * from qn;
 
 drop table t1;
+
+--echo #
+--echo # MDEV-16629: function with recursive CTE using a base table
+--echo #
+
+CREATE TABLE t1 (id int);
+INSERT INTO t1 VALUES (0), (1),(2);
+
+WITH recursive cte AS
+(SELECT id FROM t1 UNION SELECT 3 FROM cte)
+SELECT count(id) FROM cte;
+
+CREATE OR REPLACE FUNCTION func() RETURNS int
+RETURN
+(
+  WITH recursive cte AS
+  (SELECT id FROM t1 UNION SELECT 3 FROM cte)
+  SELECT count(id) FROM cte
+);
+
+SELECT func();
+
+DROP FUNCTION func;
+DROP TABLE t1;
+
+--echo #
+--echo # MDEV-16661: function with recursive CTE using no base tables
+--echo #             (fixed by the patch for MDEV-16629)
+--echo #
+
+CREATE OR REPLACE FUNCTION func() RETURNS int
+RETURN
+(
+  WITH RECURSIVE cte AS
+  (SELECT 1 as id UNION SELECT * FROM cte)
+  SELECT count(id) FROM cte
+);
+
+SELECT func();
+
+DROP FUNCTION func;
+
+--echo #
+--echo # MDEV-15151: function with recursive CTE using no base tables
+--echo #             (duplicate of  MDEV-16661)
+--echo #
+
+--connection default
+
+CREATE TABLE t1 (id int  KEY);
+INSERT INTO t1 VALUES (0), (1),(2);
+
+CREATE OR REPLACE FUNCTION func() RETURNS int
+RETURN
+(
+  WITH recursive cte AS
+  (SELECT 1 a UNION SELECT cte.* FROM cte natural join t1)
+  SELECT * FROM cte  limit 1
+);
+
+--let $conid= `SELECT CONNECTION_ID()`
+--send SELECT func();
+
+--connect (con1,localhost,root,,)
+--eval KILL QUERY $conid
+--source include/restart_mysqld.inc
+
+DROP FUNCTION func;
+DROP TABLE t1;
+--disconnect con1
+
+--connection default
diff --git a/sql/sp_head.cc b/sql/sp_head.cc
index a832aa91004..effe9d1735f 100644
--- a/sql/sp_head.cc
+++ b/sql/sp_head.cc
@@ -29,6 +29,7 @@
 #include "sql_array.h"         // Dynamic_array
 #include "log_event.h"         // Query_log_event
 #include "sql_derived.h"       // mysql_handle_derived
+#include "sql_cte.h"
 
 #ifdef USE_PRAGMA_IMPLEMENTATION
 #pragma implementation
@@ -3000,7 +3001,8 @@ sp_lex_keeper::reset_lex_and_exec_core(THD *thd, uint *nextp,
 #endif
 
   if (open_tables)
-    res= instr->exec_open_and_lock_tables(thd, m_lex->query_tables);
+    res= check_dependencies_in_with_clauses(m_lex->with_clauses_list) ||
+         instr->exec_open_and_lock_tables(thd, m_lex->query_tables);
 
   if (!res)
   {

From e9f1d8da57ba9329289c3448223cc86a13a5d918 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 5 Jul 2018 11:41:55 +0300
Subject: [PATCH 2/5] Fix warnings about possibly uninitialized variables

---
 storage/innobase/buf/buf0flu.cc    | 3 +--
 storage/innobase/dict/dict0dict.cc | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc
index 99a36fc3c45..cf53a10fa21 100644
--- a/storage/innobase/buf/buf0flu.cc
+++ b/storage/innobase/buf/buf0flu.cc
@@ -965,7 +965,7 @@ buf_flush_init_for_writing(
 		}
 	}
 
-	uint32_t checksum;
+	uint32_t checksum = BUF_NO_CHECKSUM_MAGIC;
 
 	switch (srv_checksum_algorithm_t(srv_checksum_algorithm)) {
 	case SRV_CHECKSUM_ALGORITHM_INNODB:
@@ -988,7 +988,6 @@ buf_flush_init_for_writing(
 		break;
 	case SRV_CHECKSUM_ALGORITHM_NONE:
 	case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
-		checksum = BUF_NO_CHECKSUM_MAGIC;
 		mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM,
 				checksum);
 		break;
diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index df5c678ffb5..c4b88cbea56 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -6981,6 +6981,7 @@ dict_foreign_qualify_index(
 		}
 
 		if (dict_col_is_virtual(field->col)) {
+			col_name = "";
 			for (ulint j = 0; j < table->n_v_def; j++) {
 				col_name = dict_table_get_v_col_name(table, j);
 				if (innobase_strcasecmp(field->name,col_name) == 0) {

From fdb9e66feea4ee3a8769bff74277f76e49734fc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 5 Jul 2018 13:15:35 +0300
Subject: [PATCH 3/5] Implement a parameter for wait_all_purged.inc

---
 mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test | 4 +++-
 mysql-test/suite/innodb/include/wait_all_purged.inc     | 8 +++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test b/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test
index ad733eee3a7..7feeee768ff 100644
--- a/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test
+++ b/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test
@@ -124,7 +124,7 @@ SELECT * FROM t1;
 
 DROP TABLE t1;
 
-# Test adding virutal index on existing virtual column
+# Test adding index on existing virtual column
 CREATE TABLE t1 (a INT, b INT, c INT GENERATED ALWAYS AS(a+b));
 
 INSERT INTO t1(a, b) VALUES (1, 1), (2, 2), (3, 3), (4, 4);
@@ -156,7 +156,9 @@ INSERT INTO t1(a, b) VALUES (8, 8);
 COMMIT;
 
 --echo # wait for purge to process the deleted/updated records.
+let $wait_all_purged=1;
 --source ../../innodb/include/wait_all_purged.inc
+let $wait_all_purged=0;
 
 SET DEBUG_SYNC= 'now SIGNAL purged';
 
diff --git a/mysql-test/suite/innodb/include/wait_all_purged.inc b/mysql-test/suite/innodb/include/wait_all_purged.inc
index 7dbb59a5d32..c7a16888829 100644
--- a/mysql-test/suite/innodb/include/wait_all_purged.inc
+++ b/mysql-test/suite/innodb/include/wait_all_purged.inc
@@ -1,12 +1,18 @@
 # Wait for everything to be purged.
 # The user should have set innodb_purge_rseg_truncate_frequency=1.
 
+if (!$wait_all_purged)
+{
+  let $wait_all_purged= 0;
+}
+let $remaining_expect= `select concat('InnoDB		',$wait_all_purged)`;
+
 let $wait_counter= 300;
 while ($wait_counter)
 {
     --replace_regex /.*History list length ([0-9]+).*/\1/
     let $remaining= `SHOW ENGINE INNODB STATUS`;
-    if ($remaining == 'InnoDB		0')
+    if ($remaining == $remaining_expect)
     {
         let $wait_counter= 0;
     }

From d897b4dc253d00f14c9b0b3bb94a99aadb358f6c Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Thu, 5 Jul 2018 00:07:55 +0530
Subject: [PATCH 4/5] MDEV-15855: Use atomics for dict_table_t::n_ref_count

Make dict_table_t::n_ref_count private, and protect it with
a combination of dict_sys->mutex and atomics. We want to be
able to invoke dict_table_t::release() without holding
dict_sys->mutex.
---
 storage/innobase/btr/btr0sea.cc       |  2 +-
 storage/innobase/dict/dict0dict.cc    |  4 ++--
 storage/innobase/handler/ha_innodb.cc |  4 ++--
 storage/innobase/include/dict0dict.ic | 19 +++++--------------
 storage/innobase/include/dict0mem.h   | 10 ++++++----
 storage/innobase/lock/lock0lock.cc    |  4 ++--
 6 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc
index 278062afa78..7419ed2bfd5 100644
--- a/storage/innobase/btr/btr0sea.cc
+++ b/storage/innobase/btr/btr0sea.cc
@@ -1337,7 +1337,7 @@ void btr_search_drop_page_hash_when_freed(const page_id_t& page_id)
 			/* In all our callers, the table handle should
 			be open, or we should be in the process of
 			dropping the table (preventing eviction). */
-			ut_ad(index->table->n_ref_count > 0
+			ut_ad(index->table->get_ref_count() > 0
 			      || mutex_own(&dict_sys->mutex));
 			btr_search_drop_page_hash_index(block);
 		}
diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index c4b88cbea56..8dc29863367 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -423,7 +423,7 @@ dict_table_try_drop_aborted(
 	dict_table_t*	table,		/*!< in: table, or NULL if it
 					needs to be looked up again */
 	table_id_t	table_id,	/*!< in: table identifier */
-	ulint		ref_count)	/*!< in: expected table->n_ref_count */
+	int32		ref_count)	/*!< in: expected table->n_ref_count */
 {
 	trx_t*		trx;
 
@@ -1353,7 +1353,7 @@ static
 ibool
 dict_table_can_be_evicted(
 /*======================*/
-	const dict_table_t*	table)		/*!< in: table to test */
+	dict_table_t*	table)		/*!< in: table to test */
 {
 	ut_ad(mutex_own(&dict_sys->mutex));
 	ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_X));
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 198255a30ef..7cba4ef5794 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -3058,7 +3058,7 @@ ha_innobase::update_thd(
 		   m_user_thd, thd));
 
 	/* The table should have been opened in ha_innobase::open(). */
-	DBUG_ASSERT(m_prebuilt->table->n_ref_count > 0);
+	DBUG_ASSERT(m_prebuilt->table->get_ref_count() > 0);
 
 	trx_t*	trx = check_trx_exists(thd);
 
@@ -14186,7 +14186,7 @@ ha_innobase::info_low(
 	m_prebuilt->trx->op_info = "returning various info to MariaDB";
 
 	ib_table = m_prebuilt->table;
-	DBUG_ASSERT(ib_table->n_ref_count > 0);
+	DBUG_ASSERT(ib_table->get_ref_count() > 0);
 
 	if (flag & HA_STATUS_TIME) {
 		if (is_analyze || innobase_stats_on_metadata) {
diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic
index ca2ea769612..fe2f8e32b1a 100644
--- a/storage/innobase/include/dict0dict.ic
+++ b/storage/innobase/include/dict0dict.ic
@@ -1498,23 +1498,13 @@ dict_table_is_file_per_table(
 	return !is_system_tablespace(table->space);
 }
 
-/** Get reference count.
-@return current value of n_ref_count */
-inline
-ulint
-dict_table_t::get_ref_count() const
-{
-	ut_ad(mutex_own(&dict_sys->mutex));
-	return(n_ref_count);
-}
-
 /** Acquire the table handle. */
 inline
 void
 dict_table_t::acquire()
 {
 	ut_ad(mutex_own(&dict_sys->mutex));
-	++n_ref_count;
+	my_atomic_add32_explicit(&n_ref_count, 1, MY_MEMORY_ORDER_RELAXED);
 }
 
 /** Release the table handle.
@@ -1523,9 +1513,10 @@ inline
 bool
 dict_table_t::release()
 {
-	ut_ad(mutex_own(&dict_sys->mutex));
-	ut_ad(n_ref_count > 0);
-	return !--n_ref_count;
+	int32 n = my_atomic_add32_explicit(
+			&n_ref_count, -1, MY_MEMORY_ORDER_RELAXED);
+	ut_ad(n > 0);
+	return n == 1;
 }
 
 /** Encode the number of columns and number of virtual columns in a
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
index 56f695f305d..02561176706 100644
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@@ -1357,7 +1357,11 @@ struct dict_table_t {
 
 	/** Get reference count.
 	@return current value of n_ref_count */
-	inline ulint get_ref_count() const;
+	inline int32 get_ref_count()
+	{
+		return my_atomic_load32_explicit(&n_ref_count,
+						 MY_MEMORY_ORDER_RELAXED);
+	}
 
 	/** Acquire the table handle. */
 	inline void acquire();
@@ -1737,13 +1741,11 @@ struct dict_table_t {
 	It is protected by lock_sys->mutex. */
 	ulint					n_rec_locks;
 
-#ifndef UNIV_DEBUG
 private:
-#endif
 	/** Count of how many handles are opened to this table. Dropping of the
 	table is NOT allowed until this count gets to zero. MySQL does NOT
 	itself check the number of open handles at DROP. */
-	ulint					n_ref_count;
+	int32					n_ref_count;
 
 public:
 	/** List of locks on the table. Protected by lock_sys->mutex. */
diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc
index ee0dc5a9000..1ce798fcd43 100644
--- a/storage/innobase/lock/lock0lock.cc
+++ b/storage/innobase/lock/lock0lock.cc
@@ -1493,7 +1493,7 @@ lock_rec_create_low(
 	lock_rec_bitmap_reset(lock);
 	lock_rec_set_nth_bit(lock, heap_no);
 	index->table->n_rec_locks++;
-	ut_ad(index->table->n_ref_count > 0 || !index->table->can_be_evicted);
+	ut_ad(index->table->get_ref_count() > 0 || !index->table->can_be_evicted);
 
 #ifdef WITH_WSREP
 	if (c_lock && wsrep_on_trx(trx)
@@ -3664,7 +3664,7 @@ lock_table_create(
 
 	lock->un_member.tab_lock.table = table;
 
-	ut_ad(table->n_ref_count > 0 || !table->can_be_evicted);
+	ut_ad(table->get_ref_count() > 0 || !table->can_be_evicted);
 
 	UT_LIST_ADD_LAST(trx->lock.trx_locks, lock);
 

From 1b335a74b4ea1944d6ef91113b1a510c0f11c557 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 5 Jul 2018 16:44:12 +0300
Subject: [PATCH 5/5] Clean up a test

At the end of a test, 'connection default' should be in a usable state.
This was not the case, because there was a preceding 'send' without a
'reap'. If 'reap' was added, an error would be reported because the
server was restarted after the 'send'. It is easiest to 'send' from a
separate connection and do the restart from 'connection default'.
---
 mysql-test/r/cte_recursive.result |  6 +++---
 mysql-test/t/cte_recursive.test   | 10 +++++-----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result
index 856f00abe59..d715fa20a6f 100644
--- a/mysql-test/r/cte_recursive.result
+++ b/mysql-test/r/cte_recursive.result
@@ -3314,10 +3314,10 @@ WITH recursive cte AS
 (SELECT 1 a UNION SELECT cte.* FROM cte natural join t1)
 SELECT * FROM cte  limit 1
 );
-SELECT func();;
 connect  con1,localhost,root,,;
-KILL QUERY 4;
+SELECT func();
+connection default;
+KILL QUERY 5;
 DROP FUNCTION func;
 DROP TABLE t1;
 disconnect con1;
-connection default;
diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test
index 504b2c64442..227ccd0cdb3 100644
--- a/mysql-test/t/cte_recursive.test
+++ b/mysql-test/t/cte_recursive.test
@@ -1,3 +1,4 @@
+--source include/not_embedded.inc
 create table t1 (a int, b varchar(32));
 insert into t1 values
 (4,'aaaa' ), (7,'bb'), (1,'ccc'), (4,'dd');
@@ -2342,15 +2343,14 @@ RETURN
   SELECT * FROM cte  limit 1
 );
 
---let $conid= `SELECT CONNECTION_ID()`
---send SELECT func();
-
 --connect (con1,localhost,root,,)
+--let $conid= `SELECT CONNECTION_ID()`
+--send SELECT func()
+
+--connection default
 --eval KILL QUERY $conid
 --source include/restart_mysqld.inc
 
 DROP FUNCTION func;
 DROP TABLE t1;
 --disconnect con1
-
---connection default