Bug#38522: 5 seconds delay when closing application using embedded server

The problem here is that embedded server starts handle_thread manager 
thread  on mysql_library_init() does not stop it on mysql_library_end().
At shutdown, my_thread_global_end() waits for thread count to become 0,
but since we did not stop the thread it will give up after 5 seconds.
             
Solution is to move shutdown for handle_manager thread from kill_server()
(mysqld specific) to clean_up() that is used by both embedded and mysqld.
            
This patch also contains some refactorings - to avoid duplicate code,
start_handle_manager() and stop_handle_manager() functions are introduced.
Unused variables are eliminated. handle_manager does not rely on global
variable abort_loop anymore to stop (abort_loop is not set for embedded).
            
Note: Specifically on Windows and when using DBUG version of libmysqld, 
the complete solution requires removing obsolete code my_thread_init() 
from my_thread_var(). This has a side effect that a DBUG statement 
after my_thread_end() can cause thread counter to be incremented, and 
embedded will hang for some seconds. Or worse, my_thread_init() will 
crash if critical sections have been deleted by the global cleanup 
routine that runs in a different thread. 

This patch also fixes and revert prior changes for Bug#38293 
"Libmysqld crash in mysql_library_init if language file missing".

Root cause of the crash observed in Bug#38293  was bug in my_thread_init() 
described above

client/mysql.cc

@@ -1627,8 +1627,10 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
     opt_nopager= 1;
     break;
   case OPT_MYSQL_PROTOCOL:
+#ifndef EMBEDDED_LIBRARY
     opt_protocol= find_type_or_exit(argument, &sql_protocol_typelib,
                                     opt->name);
+#endif
     break;
   case OPT_SERVER_ARG:
 #ifdef EMBEDDED_LIBRARY

dbug/dbug.c

@@ -1851,13 +1851,7 @@ static void DBUGOpenFile(CODE_STATE *cs,
       else
       {
         newfile= !EXISTS(name);
-        if (!(fp= fopen(name,
+        if (!(fp= fopen(name, append ? "a+" : "w")))
-#if defined(MSDOS) || defined(__WIN__)
-		append ? "a+c" : "wc"
-#else
-                append ? "a+" : "w"
-#endif
-		)))
         {
           (void) fprintf(stderr, ERR_OPEN, cs->process, name);
           perror("");
@@ -2407,13 +2401,4 @@ int i_am_a_dummy_function() {
        return 0;
 }
 
-#ifdef __WIN__
-char _db_doprnt_;
-char _db_enter_;
-char _db_pargs_;
-char _db_process_;
-char _db_push_;
-char _db_return_;
-#endif /*__WIN__*/
-
 #endif

include/my_dbug.h

@@ -108,22 +108,6 @@ extern FILE *_db_fp_(void);
 #define DBUG_EXPLAIN(buf,len)
 #define DBUG_EXPLAIN_INITIAL(buf,len)
 #define IF_DBUG(A)
-
-#ifdef __WIN__
-/*
-   On windows all the dll export has to be declared in the *.def file
-   so as we export these symbols in DEBUG mode we have to export
-   these in the RELEASE mode also. So below are the dummy symbols
-   for the RELEASE export
-*/
-extern  char _db_doprnt_;
-extern  char _db_enter_;
-extern  char _db_pargs_;
-extern  char _db_process_;
-extern  char _db_push_;
-extern  char _db_return_;
-#endif /*__WIN__*/
-
 #endif
 #ifdef	__cplusplus
 }

libmysql/libmysql.c

@@ -207,9 +207,7 @@ void STDCALL mysql_server_end()
   /* If library called my_init(), free memory allocated by it */
   if (!org_my_init_done)
   {
-    my_end(MY_DONT_FREE_DBUG);
+    my_end(0);
-    /* Remove TRACING, if enabled by mysql_debug() */
-    DBUG_POP();
   }
   else
   {

libmysqld/examples/CMakeLists.txt

@@ -29,13 +29,13 @@ ADD_DEFINITIONS(-DEMBEDDED_LIBRARY)
 ADD_EXECUTABLE(mysql_embedded ../../client/completion_hash.cc
                ../../client/mysql.cc ../../client/readline.cc
                ../../client/sql_string.cc)
-TARGET_LINK_LIBRARIES(mysql_embedded wsock32)
+TARGET_LINK_LIBRARIES(mysql_embedded debug dbug strings mysys vio yassl taocrypt regex ws2_32)
 ADD_DEPENDENCIES(mysql_embedded libmysqld)
 
 ADD_EXECUTABLE(mysqltest_embedded ../../client/mysqltest.c)
-TARGET_LINK_LIBRARIES(mysqltest_embedded wsock32)
+TARGET_LINK_LIBRARIES(mysqltest_embedded debug dbug strings mysys vio yassl taocrypt regex ws2_32)
 ADD_DEPENDENCIES(mysqltest_embedded libmysqld)
 
 ADD_EXECUTABLE(mysql_client_test_embedded ../../tests/mysql_client_test.c)
-TARGET_LINK_LIBRARIES(mysql_client_test_embedded wsock32)
+TARGET_LINK_LIBRARIES(mysql_client_test_embedded debug dbug strings mysys vio yassl taocrypt regex ws2_32)
 ADD_DEPENDENCIES(mysql_client_test_embedded libmysqld)

libmysqld/lib_sql.cc

@@ -386,6 +386,7 @@ static void emb_free_embedded_thd(MYSQL *mysql)
   thd->store_globals();
   thd->unlink();
   delete thd;
+  my_pthread_setspecific_ptr(THR_THD,  0);
   mysql->thd=0;
 }
 
@@ -539,12 +540,7 @@ int init_embedded_server(int argc, char **argv, char **groups)
 
   (void) thr_setconcurrency(concurrency);	// 10 by default
 
-  if (flush_time && flush_time != ~(ulong) 0L)
+  start_handle_manager();
-  {
-    pthread_t hThread;
-    if (pthread_create(&hThread,&connection_attrib,handle_manager,0))
-      sql_print_error("Warning: Can't create thread to manage maintenance");
-  }
 
   // FIXME initialize binlog_filter and rpl_filter if not already done
   //       corresponding delete is in clean_up()

libmysqld/libmysqld.def

@@ -2,94 +2,6 @@ LIBRARY		LIBMYSQLD
 DESCRIPTION	'MySQL 5.1 Embedded Server Library'
 VERSION		5.1
 EXPORTS
-	_db_process_
-	_db_enter_
-	_db_return_
-	_db_push_
-	_db_doprnt_
-	_db_pargs_
-	strnmov
-	get_charset
-	my_memmem
-	my_snprintf
-	pthread_exit
-	pthread_cond_signal
-	dynstr_append_mem
-	init_dynamic_string
-	dynstr_free
-	my_hash_free
-	my_vsnprintf
-	dynstr_append
-	my_close
-	my_open
-	dynstr_set
-	dynstr_append_os_quoted
-	my_delete
-	my_seek
-	my_write
-	create_temp_file
-	fn_format
-	dirname_part
-	my_hash_insert
-	my_hash_search
-	test_if_hard_path
-	my_copy
-	my_mkdir
-	my_sleep
-	my_strtod
-	pthread_cond_wait
-	my_strnncoll_simple
-	get_dynamic
-	my_regerror
-	init_dynamic_array2
-	pthread_create
-	pthread_cond_init
-	my_regcomp
-	my_regexec
-	my_regex_end
-	my_regfree
-	longlong2str
-	my_set_exception_pointers
-	my_print_stacktrace
-	my_thread_stack_size
-	my_safe_print_str
-	my_stat
-	_my_hash_init
-	pthread_attr_setstacksize
-	pthread_attr_init
-	my_dirend
-	wild_compare
-	my_dir
-	my_micro_time
-	find_type_or_exit
-	_dig_vec_upper
-	_dig_vec_lower
-	bmove_upp
-	delete_dynamic
-	free_defaults
-	getopt_compare_strings
-	getopt_ull_limit_value
-	handle_options
-	init_dynamic_array
-	insert_dynamic
-	int2str
-	is_prefix
-	list_add
-	list_delete
-	load_defaults
-	max_allowed_packet
-	my_cgets
-	my_end
-	my_getopt_print_errors
-	my_init
-	my_malloc
-	my_memdup
-	my_no_flags_free
-	my_path
-	my_print_help
-	my_print_variables
-	my_realloc
-	my_strdup
 	mysql_thread_end
 	mysql_thread_init
 	myodbc_remove_escape
@@ -162,47 +74,13 @@ EXPORTS
 	mysql_thread_safe
 	mysql_use_result
 	mysql_warning_count
-	set_dynamic
-	strcend
-	strcont
-	strdup_root
-	strfill
-	strinstr
-	strmake
-	strmov
-	strxmov
 	mysql_server_end
 	mysql_server_init
 	get_tty_password
-	sql_protocol_typelib
 	mysql_get_server_version
 	mysql_set_character_set
 	mysql_sqlstate
-	charsets_dir
-	disabled_my_option
-	my_charset_latin1
-	init_alloc_root
-	my_progname
-	get_charset_name
-	get_charset_by_csname
-	print_defaults
-	find_type
-	strxnmov
-	strend
-	my_fopen
-	my_fclose
-	unpack_filename
-	str2int
-	int10_to_str
-	longlong10_to_str
-	my_snprintf_8bit
-	alloc_root
-	free_root
-	my_read
-	llstr
 	mysql_get_parameters
-	mysql_thread_init
-	mysql_thread_end
 	mysql_stmt_bind_param
 	mysql_stmt_bind_result
 	mysql_stmt_execute
@@ -230,7 +108,3 @@ EXPORTS
 	mysql_stmt_attr_get
 	mysql_stmt_attr_set
 	mysql_stmt_field_count
-	get_defaults_options
-	my_charset_bin
-	my_charset_same
-	modify_defaults_file

mysys/my_thr_init.c

@@ -368,17 +368,7 @@ void my_thread_end(void)
 
 struct st_my_thread_var *_my_thread_var(void)
 {
-  struct st_my_thread_var *tmp=
+  return  my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys);
-    my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys);
-#if defined(USE_TLS)
-  /* This can only happen in a .DLL */
-  if (!tmp)
-  {
-    my_thread_init();
-    tmp=my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys);
-  }
-#endif
-  return tmp;
 }
 
 

sql/mysql_priv.h

@@ -1768,10 +1768,9 @@ int mysql_load(THD *thd, sql_exchange *ex, TABLE_LIST *table_list,
 int write_record(THD *thd, TABLE *table, COPY_INFO *info);
 
 /* sql_manager.cc */
-extern ulong volatile manager_status;
+extern bool volatile  mqh_used;
-extern bool volatile manager_thread_in_use, mqh_used;
+void start_handle_manager();
-extern pthread_t manager_thread;
+void stop_handle_manager();
-pthread_handler_t handle_manager(void *arg);
 bool mysql_manager_submit(void (*action)());
 
 

sql/mysqld.cc

@@ -784,16 +784,6 @@ static void close_connections(void)
   kill_cached_threads++;
   flush_thread_cache();
 
-  /* kill flush thread */
-  (void) pthread_mutex_lock(&LOCK_manager);
-  if (manager_thread_in_use)
-  {
-    DBUG_PRINT("quit", ("killing manager thread: 0x%lx",
-                        (ulong)manager_thread));
-   (void) pthread_cond_signal(&COND_manager);
-  }
-  (void) pthread_mutex_unlock(&LOCK_manager);
-
   /* kill connection thread */
 #if !defined(__WIN__) && !defined(__NETWARE__)
   DBUG_PRINT("quit", ("waiting for select thread: 0x%lx",
@@ -1196,6 +1186,7 @@ void clean_up(bool print_message)
   if (cleanup_done++)
     return; /* purecov: inspected */
 
+  stop_handle_manager();
   release_ddl_log();
 
   /*
@@ -4038,17 +4029,6 @@ server.");
 
 #ifndef EMBEDDED_LIBRARY
 
-static void create_maintenance_thread()
-{
-  if (flush_time && flush_time != ~(ulong) 0L)
-  {
-    pthread_t hThread;
-    if (pthread_create(&hThread,&connection_attrib,handle_manager,0))
-      sql_print_warning("Can't create thread to manage maintenance");
-  }
-}
-
-
 static void create_shutdown_thread()
 {
 #ifdef __WIN__
@@ -4363,7 +4343,7 @@ we force server id to 2, but this MySQL server will not act as a slave.");
   execute_ddl_log_recovery();
 
   create_shutdown_thread();
-  create_maintenance_thread();
+  start_handle_manager();
 
   if (Events::init(opt_noacl))
     unireg_abort(1);

sql/sql_manager.cc

@@ -23,8 +23,9 @@
 
 #include "mysql_priv.h"
 
-ulong volatile manager_status;
+
-bool volatile manager_thread_in_use;
+static bool volatile manager_thread_in_use;
+static bool abort_manager;
 
 pthread_t manager_thread;
 pthread_mutex_t LOCK_manager;
@@ -63,7 +64,6 @@ bool mysql_manager_submit(void (*action)())
 pthread_handler_t handle_manager(void *arg __attribute__((unused)))
 {
   int error = 0;
-  ulong status;
   struct timespec abstime;
   bool reset_flush_time = TRUE;
   struct handler_cb *cb= NULL;
@@ -72,7 +72,6 @@ pthread_handler_t handle_manager(void *arg __attribute__((unused)))
 
   pthread_detach_this_thread();
   manager_thread = pthread_self();
-  manager_status = 0;
   manager_thread_in_use = 1;
 
   for (;;)
@@ -87,16 +86,14 @@ pthread_handler_t handle_manager(void *arg __attribute__((unused)))
 	set_timespec(abstime, flush_time);
         reset_flush_time = FALSE;
       }
-      while (!manager_status && (!error || error == EINTR) && !abort_loop)
+      while ((!error || error == EINTR) && !abort_manager)
         error= pthread_cond_timedwait(&COND_manager, &LOCK_manager, &abstime);
     }
     else
     {
-      while (!manager_status && (!error || error == EINTR) && !abort_loop)
+      while ((!error || error == EINTR) && !abort_manager)
         error= pthread_cond_wait(&COND_manager, &LOCK_manager);
     }
-    status = manager_status;
-    manager_status = 0;
     if (cb == NULL)
     {
       cb= cb_list;
@@ -104,7 +101,7 @@ pthread_handler_t handle_manager(void *arg __attribute__((unused)))
     }
     pthread_mutex_unlock(&LOCK_manager);
 
-    if (abort_loop)
+    if (abort_manager)
       break;
 
     if (error == ETIMEDOUT || error == ETIME)
@@ -121,11 +118,42 @@ pthread_handler_t handle_manager(void *arg __attribute__((unused)))
       my_free((uchar*)cb, MYF(0));
       cb= next;
     }
-
-    if (status)
-      DBUG_PRINT("error", ("manager did not handle something: %lx", status));
   }
   manager_thread_in_use = 0;
+  DBUG_LEAVE; // Can't use DBUG_RETURN after my_thread_end
   my_thread_end();
-  DBUG_RETURN(NULL);
+  return (NULL);
 }
+
+
+/* Start handle manager thread */
+void start_handle_manager()
+{
+  DBUG_ENTER("start_handle_manager");
+  abort_manager = false;
+  if (flush_time && flush_time != ~(ulong) 0L)
+  {
+    pthread_t hThread;
+    if (pthread_create(&hThread,&connection_attrib,handle_manager,0))
+      sql_print_warning("Can't create handle_manager thread");
+  }
+  DBUG_VOID_RETURN;
+}
+
+
+/* Initiate shutdown of handle manager thread */
+void stop_handle_manager()
+{
+  DBUG_ENTER("stop_handle_manager");
+  abort_manager = true;
+  pthread_mutex_lock(&LOCK_manager);
+  if (manager_thread_in_use)
+  {
+    DBUG_PRINT("quit", ("initiate shutdown of handle manager thread: 0x%lx",
+                        (ulong)manager_thread));
+   pthread_cond_signal(&COND_manager);
+  }
+  pthread_mutex_unlock(&LOCK_manager);
+  DBUG_VOID_RETURN;
+}
+