Removed redundant service_thread_count

In contrast to thread_count, which is decremented by THD destructor,
this one was most probably intended to be decremented after all THD
destructors are done.

THD_count class was added to achieve similar effect with thread_count.

Aim is to reduce usage of LOCK_thread_count and COND_thread_count.
Part of MDEV-15135.

sql/mysqld.cc

@@ -467,7 +467,7 @@ ulong delay_key_write_options;
 uint protocol_version;
 uint lower_case_table_names;
 ulong tc_heuristic_recover= 0;
-int32 thread_count, service_thread_count;
+Atomic_counter<uint32_t> thread_count;
 int32 slave_open_temp_tables;
 ulong thread_created;
 ulong back_log, connect_timeout, concurrency, server_id;
@@ -1728,9 +1728,9 @@ static void close_connections(void)
     much smaller than even 2 seconds, this is only a safety fallback against
     stuck threads so server shutdown is not held up forever.
   */
-  DBUG_PRINT("info", ("thread_count: %d", thread_count));
+  DBUG_PRINT("info", ("thread_count: %u", uint32_t(thread_count)));
 
-  for (int i= 0; *(volatile int32*) &thread_count && i < 1000; i++)
+  for (int i= 0; thread_count && i < 1000; i++)
     my_sleep(20000);
 
   /*
@@ -1748,12 +1748,13 @@ static void close_connections(void)
   }
 #endif
   /* All threads has now been aborted */
-  DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count));
+  DBUG_PRINT("quit", ("Waiting for threads to die (count=%u)",
+                      uint32_t(thread_count)));
   mysql_mutex_lock(&LOCK_thread_count);
-  while (thread_count || service_thread_count)
+  while (thread_count)
   {
     mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
-    DBUG_PRINT("quit",("One thread died (count=%u)",thread_count));
+    DBUG_PRINT("quit",("One thread died (count=%u)", uint32_t(thread_count)));
   }
   mysql_mutex_unlock(&LOCK_thread_count);
 
@@ -2733,30 +2734,6 @@ void dec_connection_count(scheduler_functions *scheduler)
 }
 
 
-/*
-  Send a signal to unblock close_conneciton() if there is no more
-  threads running with a THD attached
-
-  It's safe to check for thread_count and service_thread_count outside
-  of a mutex as we are only interested to see if they where decremented
-  to 0 by a previous unlink_thd() call.
-
-  We should only signal COND_thread_count if both variables are 0,
-  false positives are ok.
-*/
-
-void signal_thd_deleted()
-{
-  if (!thread_count && !service_thread_count)
-  {
-    /* Signal close_connections() that all THD's are freed */
-    mysql_mutex_lock(&LOCK_thread_count);
-    mysql_cond_broadcast(&COND_thread_count);
-    mysql_mutex_unlock(&LOCK_thread_count);
-  }
-}
-
-
 /*
   Unlink thd from global list of available connections
 
@@ -8192,7 +8169,6 @@ static int mysql_init_variables(void)
   cleanup_done= 0;
   test_flags= select_errors= dropping_tables= ha_open_options=0;
   thread_count= kill_cached_threads= wake_thread= 0;
-  service_thread_count= 0;
   slave_open_temp_tables= 0;
   cached_thread_count= 0;
   opt_endinfo= using_udf_functions= 0;

sql/mysqld.h

@@ -86,7 +86,6 @@ void kill_mysql(THD *thd= 0);
 void close_connection(THD *thd, uint sql_errno= 0);
 void handle_connection_in_main_thread(CONNECT *thd);
 void create_thread_to_handle_connection(CONNECT *connect);
-void signal_thd_deleted();
 void unlink_thd(THD *thd);
 bool one_thread_per_connection_end(THD *thd, bool put_in_cache);
 void flush_thread_cache();
@@ -641,7 +640,7 @@ extern mysql_prlock_t LOCK_system_variables_hash;
 extern mysql_cond_t COND_thread_count, COND_start_thread;
 extern mysql_cond_t COND_manager;
 extern mysql_cond_t COND_slave_background;
-extern int32 thread_count, service_thread_count;
+extern Atomic_counter<uint32_t> thread_count;
 
 extern char *opt_ssl_ca, *opt_ssl_capath, *opt_ssl_cert, *opt_ssl_cipher,
   *opt_ssl_key, *opt_ssl_crl, *opt_ssl_crlpath;

sql/semisync_master_ack_receiver.cc

@@ -205,7 +205,6 @@ void Ack_receiver::run()
   thd->thread_stack= (char*) &thd;
   thd->store_globals();
   thd->security_ctx->skip_grants();
-  thread_safe_increment32(&service_thread_count);
   thd->set_command(COM_DAEMON);
   init_net(&net, net_buff, REPLY_MESSAGE_MAX_LENGTH);
 
@@ -284,8 +283,6 @@ end:
   sql_print_information("Stopping ack receiver thread");
   m_status= ST_DOWN;
   delete thd;
-  thread_safe_decrement32(&service_thread_count);
-  signal_thd_deleted();
   mysql_cond_broadcast(&m_cond);
   mysql_mutex_unlock(&m_mutex);
   DBUG_VOID_RETURN;

sql/slave.cc

@@ -482,7 +482,6 @@ handle_slave_background(void *arg __attribute__((unused)))
   thd= new THD(next_thread_id());
   thd->thread_stack= (char*) &thd;           /* Set approximate stack start */
   thd->system_thread = SYSTEM_THREAD_SLAVE_BACKGROUND;
-  thread_safe_increment32(&service_thread_count);
   thd->store_globals();
   thd->security_ctx->skip_grants();
   thd->set_command(COM_DAEMON);
@@ -568,8 +567,6 @@ handle_slave_background(void *arg __attribute__((unused)))
   mysql_mutex_unlock(&LOCK_slave_background);
 
   delete thd;
-  thread_safe_decrement32(&service_thread_count);
-  signal_thd_deleted();
 
   my_thread_end();
   return 0;
@@ -3578,7 +3575,6 @@ static int init_slave_thread(THD* thd, Master_info *mi,
 
   thd->system_thread = (thd_type == SLAVE_THD_SQL) ?
     SYSTEM_THREAD_SLAVE_SQL : SYSTEM_THREAD_SLAVE_IO;
-  thread_safe_increment32(&service_thread_count);
 
   /* We must call store_globals() before doing my_net_init() */
   if (init_thr_lock() || thd->store_globals() ||
@@ -5092,8 +5088,6 @@ err_during_init:
 
   thd->assert_not_linked();
   delete thd;
-  thread_safe_decrement32(&service_thread_count);
-  signal_thd_deleted();
 
   mi->abort_slave= 0;
   mi->slave_running= MYSQL_SLAVE_NOT_RUN;
@@ -5786,8 +5780,6 @@ err_during_init:
 
   delete serial_rgi;
   delete thd;
-  thread_safe_decrement32(&service_thread_count);
-  signal_thd_deleted();
 
   DBUG_LEAVE;                                   // Must match DBUG_ENTER()
   my_thread_end();

sql/sql_class.cc

@@ -864,7 +864,6 @@ THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock)
   save_prep_leaf_list= FALSE;
   /* Restore THR_THD */
   set_current_thd(old_THR_THD);
-  inc_thread_count();
 }
 
 
@@ -1723,7 +1722,6 @@ THD::~THD()
   }
   update_global_memory_status(status_var.global_memory_used);
   set_current_thd(orig_thd == this ? 0 : orig_thd);
-  dec_thread_count();
   DBUG_VOID_RETURN;
 }
 

sql/sql_class.h

@@ -2134,13 +2134,46 @@ struct wait_for_commit
 
 extern "C" void my_message_sql(uint error, const char *str, myf MyFlags);
 
+
+/**
+  A wrapper around thread_count.
+
+  It must be specified as a first base class of THD, so that increment is
+  done before any other THD constructors and decrement - after any other THD
+  destructors.
+*/
+struct THD_count
+{
+  THD_count() { thread_count++; }
+
+
+  /**
+    Decrements thread_count.
+
+    Unblocks close_conneciton() if there are no more THD's left.
+  */
+  ~THD_count()
+  {
+    uint32_t t= thread_count--;
+    DBUG_ASSERT(t > 0);
+    if (t == 1)
+    {
+      mysql_mutex_lock(&LOCK_thread_count);
+      mysql_cond_broadcast(&COND_thread_count);
+      mysql_mutex_unlock(&LOCK_thread_count);
+    }
+  }
+};
+
+
 /**
   @class THD
   For each client connection we create a separate thread with THD serving as
   a thread/connection descriptor
 */
 
-class THD :public Statement,
+class THD: public THD_count, /* this must be first */
+           public Statement,
            /*
              This is to track items changed during execution of a prepared
              statement/stored procedure. It's created by
@@ -2165,19 +2198,6 @@ private:
   inline bool is_conventional() const
   { DBUG_ASSERT(0); return Statement::is_conventional(); }
 
-  void dec_thread_count(void)
-  {
-    DBUG_ASSERT(thread_count > 0);
-    thread_safe_decrement32(&thread_count);
-    signal_thd_deleted();
-  }
-
-
-  void inc_thread_count(void)
-  {
-    thread_safe_increment32(&thread_count);
-  }
-
 public:
   MDL_context mdl_context;
 

sql/wsrep_mysqld.cc

@@ -2311,13 +2311,15 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD* except_caller_thd)
   */
   server_threads.iterate(kill_remaining_threads, except_caller_thd);
 
-  DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count));
+  DBUG_PRINT("quit", ("Waiting for threads to die (count=%u)",
-  WSREP_DEBUG("waiting for client connections to close: %u", thread_count);
+             uint32_t(thread_count)));
+  WSREP_DEBUG("waiting for client connections to close: %u",
+              uint32_t(thread_count));
 
   while (wait_to_end && server_threads.iterate(have_client_connections))
   {
     mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
-    DBUG_PRINT("quit",("One thread died (count=%u)", thread_count));
+    DBUG_PRINT("quit",("One thread died (count=%u)", uint32_t(thread_count)));
   }
 
   kill_cached_threads= kill_cached_threads_saved;
@@ -2371,7 +2373,7 @@ void wsrep_wait_appliers_close(THD *thd)
     }
     else
       mysql_cond_wait(&COND_thread_count,&LOCK_thread_count);
-    DBUG_PRINT("quit",("One applier died (count=%u)",thread_count));
+    DBUG_PRINT("quit",("One applier died (count=%u)", uint32_t(thread_count)));
   }
   mysql_mutex_unlock(&LOCK_thread_count);
   /* Now kill remaining wsrep threads: rollbacker */
@@ -2388,7 +2390,7 @@ void wsrep_wait_appliers_close(THD *thd)
     }
     else
       mysql_cond_wait(&COND_thread_count,&LOCK_thread_count);
-    DBUG_PRINT("quit",("One thread died (count=%u)",thread_count));
+    DBUG_PRINT("quit",("One thread died (count=%u)", uint32_t(thread_count)));
   }
   mysql_mutex_unlock(&LOCK_thread_count);