MDEV-21102: Server crashes in JOIN_CACHE::write_record_data upon EXPLAIN with subqueries

JOIN_CACHE has a light-weight initialization mode that's targeted at
EXPLAINs. In that mode, JOIN_CACHE objects are not able to execute.

Light-weight mode was used whenever the statement was an EXPLAIN. However
the EXPLAIN can execute subqueries, provided they enumerate less than
@@expensive_subquery_limit rows.

Make sure we use light-weight initialization mode only when the select is
more expensive @@expensive_subquery_limit.

Also add an assert into JOIN_CACHE::put_record() which prevents its use
if it was initialized for EXPLAIN only.

mysql-test/main/join_cache.result

@@ -6396,5 +6396,27 @@ b	b	d	c	c
 10	NULL	NULL	NULL	NULL
 DROP TABLE t1,t2,t3,t4;
 #
+# MDEV-21102: Server crashes in JOIN_CACHE::write_record_data upon EXPLAIN with subqueries and constant tables
+#
+CREATE TABLE t1 (a int, b int) ENGINE=MyISAM;
+CREATE TABLE t2 (c int, d int) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1,10);
+CREATE TABLE t3 (e int, key (e)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (2),(3);
+# Must not crash, must use join buffer in subquery
+EXPLAIN
+SELECT * FROM t1
+WHERE a > b OR a IN (
+SELECT c FROM t2 WHERE EXISTS (
+SELECT * FROM t3 t3a JOIN t3 t3b WHERE t3a.e < d
+)
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
+2	DEPENDENT SUBQUERY	t2	system	NULL	NULL	NULL	NULL	1	
+3	SUBQUERY	t3a	index	e	e	5	NULL	2	Using where; Using index
+3	SUBQUERY	t3b	index	NULL	e	5	NULL	2	Using index; Using join buffer (flat, BNL join)
+DROP TABLE t1,t2,t3;
+#
 # End of 10.4 tests
 #

mysql-test/main/join_cache.test

@@ -4305,6 +4305,27 @@ eval $q2;
 
 DROP TABLE t1,t2,t3,t4;
 
+--echo #
+--echo # MDEV-21102: Server crashes in JOIN_CACHE::write_record_data upon EXPLAIN with subqueries and constant tables
+--echo #
+CREATE TABLE t1 (a int, b int) ENGINE=MyISAM;
+
+CREATE TABLE t2 (c int, d int) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1,10);
+
+CREATE TABLE t3 (e int, key (e)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (2),(3);
+
+--echo # Must not crash, must use join buffer in subquery
+EXPLAIN
+SELECT * FROM t1
+WHERE a > b OR a IN (
+    SELECT c FROM t2 WHERE EXISTS (
+        SELECT * FROM t3 t3a JOIN t3 t3b WHERE t3a.e < d
+    )
+);
+DROP TABLE t1,t2,t3;
+
 --echo #
 --echo # End of 10.4 tests
 --echo #

sql/item_subselect.cc

@@ -562,6 +562,8 @@ void Item_subselect::recalc_used_tables(st_select_lex *new_parent,
   This measure is used instead of JOIN::read_time, because it is considered
   to be much more reliable than the cost estimate.
 
+  Note: the logic in this function must agree with JOIN::init_join_caches().
+
   @return true if the subquery is expensive
   @return false otherwise
 */

sql/sql_join_cache.cc

@@ -1589,6 +1589,7 @@ bool JOIN_CACHE::put_record()
 {
   bool is_full;
   uchar *link= 0;
+  DBUG_ASSERT(!for_explain_only);
   if (prev_cache)
     link= prev_cache->get_curr_rec_link();
   write_record_data(link, &is_full);

sql/sql_select.cc

@@ -1876,6 +1876,26 @@ JOIN::init_range_rowid_filters()
 
 int JOIN::init_join_caches()
 {
+  bool init_for_explain= false;
+
+  /*
+    Can we use lightweight initalization mode just for EXPLAINs? We can if
+    we're certain that the optimizer will not execute the subquery.
+    The optimzier will not execute the subquery if it's too expensive. For
+    the exact criteria, see Item_subselect::is_expensive(). 
+    Note that the subquery might be a UNION and we might not yet know if it is
+    expensive.
+    What we do know is that if this SELECT is too expensive, then the whole 
+    subquery will be too expensive as well.
+    So, we can use lightweight initialization (init_for_explain=true) if this
+    SELECT examines more than @@expensive_subquery_limit rows.
+  */
+  if ((select_options & SELECT_DESCRIBE) &&
+      get_examined_rows() >= thd->variables.expensive_subquery_limit)
+  {
+    init_for_explain= true;
+  }
+
   JOIN_TAB *tab;
 
   for (tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES);
@@ -1895,7 +1915,8 @@ int JOIN::init_join_caches()
     {
       table->prepare_for_keyread(tab->index, table->read_set);
     }
-    if (tab->cache && tab->cache->init(select_options & SELECT_DESCRIBE))
+
+    if (tab->cache && tab->cache->init(init_for_explain))
       revise_cache_usage(tab);
     else
       tab->remove_redundant_bnl_scan_conds();