MDEV-5616 - Deadlock between CREATE/DROP FUNCTION and SELECT from view

Deadlock happened due to mixed lock order.
CREATE/DROP function: wrlock(THR_LOCK_udf) -> lock(LOCK_open)
SELECT from view: lock(LOCK_open) -> rdlock(THR_LOCK_udf)

Fixed CREATE/DROP function so that LOCK_open does not intersect with
wrlock(THR_LOCK_udf).

10.0 is not affected: it doesn't hold LOCK_open while opening view.

mysql-test/r/udf_debug_sync.result

@@ -0,0 +1,20 @@
+CREATE FUNCTION myfunc_int RETURNS INTEGER SONAME "UDF_EXAMPLE_LIB";
+CREATE VIEW v1 AS SELECT myfunc_int(1);
+SET debug_sync='mysql_create_function_after_lock SIGNAL locked WAIT_FOR go';
+CREATE FUNCTION myfunc_double RETURNS REAL SONAME "UDF_EXAMPLE_LIB";
+SET debug_sync='now WAIT_FOR locked';
+SET debug_sync='find_udf_before_lock SIGNAL go';
+SELECT * FROM v1;
+myfunc_int(1)
+1
+FLUSH TABLES;
+SET debug_sync='mysql_drop_function_after_lock SIGNAL locked WAIT_FOR go';
+DROP FUNCTION myfunc_double;
+SET debug_sync='now WAIT_FOR locked';
+SET debug_sync='find_udf_before_lock SIGNAL go';
+SELECT * FROM v1;
+myfunc_int(1)
+1
+SET debug_sync='RESET';
+DROP VIEW v1;
+DROP FUNCTION myfunc_int;

mysql-test/t/udf_debug_sync.test

@@ -0,0 +1,40 @@
+--source include/have_debug_sync.inc
+--source include/have_udf.inc
+
+#
+# MDEV-5616 - Deadlock between CREATE/DROP FUNCTION and SELECT from view
+#
+--replace_result $UDF_EXAMPLE_SO UDF_EXAMPLE_LIB
+eval CREATE FUNCTION myfunc_int RETURNS INTEGER SONAME "$UDF_EXAMPLE_SO";
+CREATE VIEW v1 AS SELECT myfunc_int(1);
+connect(con1, localhost, root,,);
+
+connection con1;
+SET debug_sync='mysql_create_function_after_lock SIGNAL locked WAIT_FOR go';
+--replace_result $UDF_EXAMPLE_SO UDF_EXAMPLE_LIB
+send_eval CREATE FUNCTION myfunc_double RETURNS REAL SONAME "$UDF_EXAMPLE_SO";
+
+connection default;
+SET debug_sync='now WAIT_FOR locked';
+SET debug_sync='find_udf_before_lock SIGNAL go';
+SELECT * FROM v1;
+FLUSH TABLES;
+
+connection con1;
+reap;
+SET debug_sync='mysql_drop_function_after_lock SIGNAL locked WAIT_FOR go';
+send DROP FUNCTION myfunc_double;
+
+connection default;
+SET debug_sync='now WAIT_FOR locked';
+SET debug_sync='find_udf_before_lock SIGNAL go';
+SELECT * FROM v1;
+
+connection con1;
+reap;
+disconnect con1;
+
+connection default;
+SET debug_sync='RESET';
+DROP VIEW v1;
+DROP FUNCTION myfunc_int;

sql/sql_udf.cc

@@ -352,6 +352,7 @@ udf_func *find_udf(const char *name,uint length,bool mark_used)
   if (!initialized)
     DBUG_RETURN(NULL);
 
+  DEBUG_SYNC(current_thd, "find_udf_before_lock");
   /* TODO: This should be changed to reader locks someday! */
   if (mark_used)
     mysql_rwlock_wrlock(&THR_LOCK_udf);  /* Called during fix_fields */
@@ -466,7 +467,12 @@ int mysql_create_function(THD *thd,udf_func *udf)
   if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row()))
     thd->clear_current_stmt_binlog_format_row();
 
+  tables.init_one_table(STRING_WITH_LEN("mysql"), STRING_WITH_LEN("func"),
+                        "func", TL_WRITE);
+  table= open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT);
+
   mysql_rwlock_wrlock(&THR_LOCK_udf);
+  DEBUG_SYNC(current_thd, "mysql_create_function_after_lock");
   if ((my_hash_search(&udf_hash,(uchar*) udf->name.str, udf->name.length)))
   {
     my_error(ER_UDF_EXISTS, MYF(0), udf->name.str);
@@ -510,9 +516,8 @@ int mysql_create_function(THD *thd,udf_func *udf)
 
   /* create entry in mysql.func table */
 
-  tables.init_one_table("mysql", 5, "func", 4, "func", TL_WRITE);
   /* Allow creation of functions even if we can't open func table */
-  if (!(table = open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT)))
+  if (!table)
     goto err;
   table->use_all_columns();
   restore_record(table, s->default_values);	// Default values for fields
@@ -584,7 +589,12 @@ int mysql_drop_function(THD *thd,const LEX_STRING *udf_name)
   if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row()))
     thd->clear_current_stmt_binlog_format_row();
 
+  tables.init_one_table(STRING_WITH_LEN("mysql"), STRING_WITH_LEN("func"),
+                        "func", TL_WRITE);
+  table= open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT);
+
   mysql_rwlock_wrlock(&THR_LOCK_udf);
+  DEBUG_SYNC(current_thd, "mysql_drop_function_after_lock");
   if (!(udf=(udf_func*) my_hash_search(&udf_hash,(uchar*) udf_name->str,
                                        (uint) udf_name->length)))
   {
@@ -601,9 +611,7 @@ int mysql_drop_function(THD *thd,const LEX_STRING *udf_name)
   if (udf->dlhandle && !find_udf_dl(udf->dl))
     dlclose(udf->dlhandle);
 
-  tables.init_one_table("mysql", 5, "func", 4, "func", TL_WRITE);
+  if (!table)
-
-  if (!(table = open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT)))
     goto err;
   table->use_all_columns();
   table->field[0]->store(exact_name_str, exact_name_len, &my_charset_bin);