diff --git a/include/haproxy/task.h b/include/haproxy/task.h index b9e733f45..e31e9dac4 100644 --- a/include/haproxy/task.h +++ b/include/haproxy/task.h @@ -91,6 +91,8 @@ extern struct pool_head *pool_head_task; extern struct pool_head *pool_head_tasklet; extern struct pool_head *pool_head_notification; +__decl_thread(extern HA_RWLOCK_T wq_lock THREAD_ALIGNED(64)); + void __tasklet_wakeup_on(struct tasklet *tl, int thr); struct list *__tasklet_wakeup_after(struct list *head, struct tasklet *tl); void task_kill(struct task *t); @@ -264,10 +266,10 @@ static inline struct task *task_unlink_wq(struct task *t) locked = t->tid < 0; BUG_ON(t->tid >= 0 && t->tid != tid && !(global.mode & MODE_STOPPING)); if (locked) - HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock); __task_unlink_wq(t); if (locked) - HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock); } return t; } @@ -294,10 +296,10 @@ static inline void task_queue(struct task *task) #ifdef USE_THREAD if (task->tid < 0) { - HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock); if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key)) __task_queue(task, &tg_ctx->timers); - HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock); } else #endif { @@ -654,14 +656,14 @@ static inline void task_schedule(struct task *task, int when) #ifdef USE_THREAD if (task->tid < 0) { /* FIXME: is it really needed to lock the WQ during the check ? */ - HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock); if (task_in_wq(task)) when = tick_first(when, task->expire); task->expire = when; if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key)) __task_queue(task, &tg_ctx->timers); - HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock); } else #endif { diff --git a/include/haproxy/tinfo-t.h b/include/haproxy/tinfo-t.h index 10fe7e988..4c2cef51c 100644 --- a/include/haproxy/tinfo-t.h +++ b/include/haproxy/tinfo-t.h @@ -70,7 +70,6 @@ struct tgroup_ctx { ulong threads_idle; /* mask of threads idling in the poller */ ulong stopping_threads; /* mask of threads currently stopping */ - HA_RWLOCK_T wq_lock; /* RW lock related to the wait queue below */ struct eb_root timers; /* wait queue (sorted timers tree, global, accessed under wq_lock) */ uint niced_tasks; /* number of niced tasks in this group's run queues */ diff --git a/src/task.c b/src/task.c index b7a3f0199..5d009faf3 100644 --- a/src/task.c +++ b/src/task.c @@ -35,6 +35,12 @@ DECLARE_POOL(pool_head_tasklet, "tasklet", sizeof(struct tasklet)); */ DECLARE_POOL(pool_head_notification, "notification", sizeof(struct notification)); +/* The lock protecting all wait queues at once. For now we have no better + * alternative since a task may have to be removed from a queue and placed + * into another one. Storing the WQ index into the task doesn't seem to be + * sufficient either. + */ +__decl_thread(HA_RWLOCK_T wq_lock THREAD_ALIGNED(64) = 0); /* Flags the task for immediate destruction and puts it into its first * thread's shared tasklet list if not yet queued/running. This will bypass @@ -362,29 +368,29 @@ void wake_expired_tasks() if (eb_is_empty(&tg_ctx->timers)) goto leave; - HA_RWLOCK_RDLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_RDLOCK(TASK_WQ_LOCK, &wq_lock); eb = eb32_lookup_ge(&tg_ctx->timers, now_ms - TIMER_LOOK_BACK); if (!eb) { eb = eb32_first(&tg_ctx->timers); if (likely(!eb)) { - HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &wq_lock); goto leave; } } key = eb->key; if (tick_is_lt(now_ms, key)) { - HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &wq_lock); goto leave; } /* There's really something of interest here, let's visit the queue */ - if (HA_RWLOCK_TRYRDTOSK(TASK_WQ_LOCK, &tg_ctx->wq_lock)) { + if (HA_RWLOCK_TRYRDTOSK(TASK_WQ_LOCK, &wq_lock)) { /* if we failed to grab the lock it means another thread is * already doing the same here, so let it do the job. */ - HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &wq_lock); goto leave; } @@ -423,20 +429,20 @@ void wake_expired_tasks() if (tick_is_expired(task->expire, now_ms)) { /* expired task, wake it up */ - HA_RWLOCK_SKTOWR(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_SKTOWR(TASK_WQ_LOCK, &wq_lock); __task_unlink_wq(task); - HA_RWLOCK_WRTOSK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_WRTOSK(TASK_WQ_LOCK, &wq_lock); task_drop_running(task, TASK_WOKEN_TIMER); } else if (task->expire != eb->key) { /* task is not expired but its key doesn't match so let's * update it and skip to next apparently expired task. */ - HA_RWLOCK_SKTOWR(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_SKTOWR(TASK_WQ_LOCK, &wq_lock); __task_unlink_wq(task); if (tick_isset(task->expire)) __task_queue(task, &tg_ctx->timers); - HA_RWLOCK_WRTOSK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_WRTOSK(TASK_WQ_LOCK, &wq_lock); task_drop_running(task, 0); goto lookup_next; } @@ -448,7 +454,7 @@ void wake_expired_tasks() } } - HA_RWLOCK_SKUNLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_SKUNLOCK(TASK_WQ_LOCK, &wq_lock); #endif leave: return; @@ -480,13 +486,13 @@ int next_timer_expiry() #ifdef USE_THREAD if (!eb_is_empty(&tg_ctx->timers)) { - HA_RWLOCK_RDLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_RDLOCK(TASK_WQ_LOCK, &wq_lock); eb = eb32_lookup_ge(&tg_ctx->timers, now_ms - TIMER_LOOK_BACK); if (!eb) eb = eb32_first(&tg_ctx->timers); if (eb) key = eb->key; - HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &tg_ctx->wq_lock); + HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &wq_lock); if (eb) ret = tick_first(ret, key); }