MDEV-34142 Server crashes in create_internal_tmp_table with low tmp space limit

mysql-test/main/tmp_space_usage.result

@@ -213,4 +213,25 @@ ERROR HY000: Local temporary space limit reached
 drop table t1;
 connection default;
 disconnect c1;
+#
+# MDEV-34142 Server crashes in create_internal_tmp_table with low tmp
+# space limit
+#
+SET MAX_TMP_SESSION_SPACE_USAGE = 128*1024, MAX_HEAP_TABLE_SIZE= 16*1024*1024;
+CREATE TABLE t1 (a varchar(1024)) DEFAULT CHARACTER SET utf8mb3;
+INSERT INTO t1 SELECT 'x' FROM seq_1_to_50;
+SELECT * FROM t1 JOIN seq_1_to_200 INTERSECT ALL SELECT * FROM t1 JOIN seq_1_to_200;
+ERROR HY000: Local temporary space limit reached
+drop table t1;
+#
+# MDEV-34149 Corruption-like errors are produced when temporary space
+# limit is reached
+#
+SET MAX_TMP_SESSION_SPACE_USAGE = 400*1024;
+Warnings:
+Warning	1292	Truncated incorrect max_tmp_session_space_usage value: '409600'
+SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_VARIABLES JOIN seq_1_to_100
+INTERSECT ALL
+SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_VARIABLES JOIN seq_1_to_100;
+ERROR HY000: Local temporary space limit reached
 # End of 11.5 tests

mysql-test/main/tmp_space_usage.test

@@ -275,4 +275,27 @@ drop table t1;
 connection default;
 disconnect c1;
 
+--echo #
+--echo # MDEV-34142 Server crashes in create_internal_tmp_table with low tmp
+--echo # space limit
+--echo #
+
+SET MAX_TMP_SESSION_SPACE_USAGE = 128*1024, MAX_HEAP_TABLE_SIZE= 16*1024*1024;
+CREATE TABLE t1 (a varchar(1024)) DEFAULT CHARACTER SET utf8mb3;
+INSERT INTO t1 SELECT 'x' FROM seq_1_to_50;
+--error 200
+SELECT * FROM t1 JOIN seq_1_to_200 INTERSECT ALL SELECT * FROM t1 JOIN seq_1_to_200;
+drop table t1;
+
+--echo #
+--echo # MDEV-34149 Corruption-like errors are produced when temporary space
+--echo # limit is reached
+--echo #
+
+SET MAX_TMP_SESSION_SPACE_USAGE = 400*1024;
+--error 200
+SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_VARIABLES JOIN seq_1_to_100
+INTERSECT ALL
+SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_VARIABLES JOIN seq_1_to_100;
+
 --echo # End of 11.5 tests

sql/sql_select.cc

@@ -21252,7 +21252,7 @@ TABLE *Create_tmp_table::start(THD *thd,
                         &m_key_part_info,
                         sizeof(*m_key_part_info)*(param->group_parts+1),
                         &param->start_recinfo,
-                        sizeof(*param->recinfo)*(field_count*2+4),
+                        sizeof(*param->start_recinfo)*(field_count*2+4),
                         &param->rec_per_key, sizeof(ulong)*param->group_parts,
                         &tmpname, (uint) strlen(path)+1,
                         &m_group_buff, (m_group && ! m_using_unique_constraint ?
@@ -22714,6 +22714,7 @@ bool create_internal_tmp_table(TABLE *table, KEY *org_keyinfo,
   and copy all rows to this
 
   In case of error, my_error() or handler::print_error() will be called.
+  Note that in case of error, table->file->ha_rnd_end() may have been called!
 */
 
 

sql/sql_union.cc

@@ -157,8 +157,9 @@ int select_unit::send_data(List<Item> &values)
   switch (step)
   {
   case UNION_TYPE:
+    /* Errors not related to duplicate key are reported by write_record() */
     rc= write_record();
-    /* no reaction with conversion */
+    /* no reaction with conversion. rc == -1 (dupp key) is ignored by caller */
     if (rc == -2)
       rc= 0;
     break;
@@ -431,19 +432,12 @@ int select_unit::write_record()
                                               tmp_table_param.start_recinfo,
                                               &tmp_table_param.recinfo,
                                               write_err, 1, &is_duplicate))
-      {
         return -2;
-      }
-      else
-      {
       return 1;
     }
-    }
     if (is_duplicate)
-    {
       return -1;
   }
-  }
   return 0;
 }
 
@@ -498,27 +492,26 @@ bool select_unit_ext::disable_index_if_needed(SELECT_LEX *curr_sl)
   @retval
     0   no error
     -1  conversion happened
+    1   error
+
+    Note that duplicate keys are ignored (write_record() is returning -1)
 */
 
 int select_unit_ext::unfold_record(ha_rows cnt)
 {
 
   DBUG_ASSERT(cnt > 0);
-  int error= 0;
+  int ret= 0;
-  bool is_convertion_happened= false;
   while (--cnt)
   {
-    error= write_record();
+    int error= write_record();
     if (error == -2)
-    {
+      ret= -1;                                  // Conversion happened
-      is_convertion_happened= true;
+    else if (error > 0)
-      error= -1;
-    }
-  }
-  if (is_convertion_happened)
-    return -1;
       return error;
   }
+  return ret;
+}
 
 /*
   @brief
@@ -873,7 +866,7 @@ bool select_unit_ext::send_eof()
       if (unlikely(error))
         break;
 
-      if (unfold_record(dup_cnt) == -1)
+      if ((error= unfold_record(dup_cnt)) == -1)
       {
         /* restart the scan */
         if (unlikely(table->file->ha_rnd_init_with_error(1)))
@@ -884,8 +877,14 @@ bool select_unit_ext::send_eof()
           additional_cnt= table->field[addon_cnt - 2];
         else
           additional_cnt= NULL;
+        error= 0;
         continue;
       }
+      else if (error > 0)
+      {
+        table->file->ha_index_or_rnd_end();
+        return 1;
+      }
     } while (likely(!error));
     table->file->ha_rnd_end();
   }