[PATCH 23/31] workqueue: implement get/put_pwq()
Add pool_workqueue->refcnt along with get/put_pwq(). Both per-cpu and
unbound pwqs have refcnts and any work item inserted on a pwq
increments the refcnt which is dropped when the work item finishes.
For per-cpu pwqs the base ref is never dropped and destroy_workqueue()
frees the pwqs as before. For unbound ones, destroy_workqueue()
simply drops the base ref on the first pwq. When the refcnt reaches
zero, pwq_unbound_release_workfn() is scheduled on system_wq, which
unlinks the pwq, puts the associated pool and frees the pwq and wq as
necessary. This needs to be done from a work item as put_pwq() needs
to be protected by pool->lock but release can't happen with the lock
held - e.g. put_unbound_pool() involves blocking operations.
Unbound pool->locks are marked with lockdep subclas 1 as put_pwq()
will schedule the release work item on system_wq while holding the
unbound pool's lock and triggers recursive locking warning spuriously.
This will be used to implement dynamic creation and destruction of
unbound pwqs.
Signed-off-by: Tejun Heo
---
kernel/workqueue.c | 137 -
1 file changed, 114 insertions(+), 23 deletions(-)
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index d0604ee..e092cd5 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -179,6 +179,7 @@ struct pool_workqueue {
struct workqueue_struct *wq;/* I: the owning workqueue */
int work_color; /* L: current color */
int flush_color;/* L: flushing color */
+ int refcnt; /* L: reference count */
int nr_in_flight[WORK_NR_COLORS];
/* L: nr of in_flight works */
int nr_active; /* L: nr of active works */
@@ -186,6 +187,15 @@ struct pool_workqueue {
struct list_headdelayed_works; /* L: delayed works */
struct list_headpwqs_node; /* R: node on wq->pwqs */
struct list_headmayday_node;/* W: node on wq->maydays */
+
+ /*
+* Release of unbound pwq is punted to system_wq. See put_pwq()
+* and pwq_unbound_release_workfn() for details. pool_workqueue
+* itself is also sched-RCU protected so that the first pwq can be
+* determined without grabbing workqueue_lock.
+*/
+ struct work_struct unbound_release_work;
+ struct rcu_head rcu;
} __aligned(1 << WORK_STRUCT_FLAG_BITS);
/*
@@ -936,6 +946,45 @@ static void move_linked_works(struct work_struct *work,
struct list_head *head,
*nextp = n;
}
+/**
+ * get_pwq - get an extra reference on the specified pool_workqueue
+ * @pwq: pool_workqueue to get
+ *
+ * Obtain an extra reference on @pwq. The caller should guarantee that
+ * @pwq has positive refcnt and be holding the matching pool->lock.
+ */
+static void get_pwq(struct pool_workqueue *pwq)
+{
+ lockdep_assert_held(>pool->lock);
+ WARN_ON_ONCE(pwq->refcnt <= 0);
+ pwq->refcnt++;
+}
+
+/**
+ * put_pwq - put a pool_workqueue reference
+ * @pwq: pool_workqueue to put
+ *
+ * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
+ * destruction. The caller should be holding the matching pool->lock.
+ */
+static void put_pwq(struct pool_workqueue *pwq)
+{
+ lockdep_assert_held(>pool->lock);
+ if (likely(--pwq->refcnt))
+ return;
+ if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
+ return;
+ /*
+* @pwq can't be released under pool->lock, bounce to
+* pwq_unbound_release_workfn(). This never recurses on the same
+* pool->lock as this path is taken only for unbound workqueues and
+* the release work item is scheduled on a per-cpu workqueue. To
+* avoid lockdep warning, unbound pool->locks are given lockdep
+* subclass of 1 in get_unbound_pool().
+*/
+ schedule_work(>unbound_release_work);
+}
+
static void pwq_activate_delayed_work(struct work_struct *work)
{
struct pool_workqueue *pwq = get_work_pwq(work);
@@ -967,9 +1016,9 @@ static void pwq_activate_first_delayed(struct
pool_workqueue *pwq)
*/
static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
{
- /* ignore uncolored works */
+ /* uncolored work items don't participate in flushing or nr_active */
if (color == WORK_NO_COLOR)
- return;
+ goto out_put;
pwq->nr_in_flight[color]--;
@@ -982,11 +1031,11 @@ static void pwq_dec_nr_in_flight(struct pool_workqueue
*pwq, int color)
/* is flush in progress and are we at the flushing tip? */
if (likely(pwq->flush_color != color))
- return;
+ goto out_put;
/* are there still in-flight works? */
if (pwq->nr_in_flight[color])
-
[PATCH 23/31] workqueue: implement get/put_pwq()
Add pool_workqueue-refcnt along with get/put_pwq(). Both per-cpu and
unbound pwqs have refcnts and any work item inserted on a pwq
increments the refcnt which is dropped when the work item finishes.
For per-cpu pwqs the base ref is never dropped and destroy_workqueue()
frees the pwqs as before. For unbound ones, destroy_workqueue()
simply drops the base ref on the first pwq. When the refcnt reaches
zero, pwq_unbound_release_workfn() is scheduled on system_wq, which
unlinks the pwq, puts the associated pool and frees the pwq and wq as
necessary. This needs to be done from a work item as put_pwq() needs
to be protected by pool-lock but release can't happen with the lock
held - e.g. put_unbound_pool() involves blocking operations.
Unbound pool-locks are marked with lockdep subclas 1 as put_pwq()
will schedule the release work item on system_wq while holding the
unbound pool's lock and triggers recursive locking warning spuriously.
This will be used to implement dynamic creation and destruction of
unbound pwqs.
Signed-off-by: Tejun Heo t...@kernel.org
---
kernel/workqueue.c | 137 -
1 file changed, 114 insertions(+), 23 deletions(-)
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index d0604ee..e092cd5 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -179,6 +179,7 @@ struct pool_workqueue {
struct workqueue_struct *wq;/* I: the owning workqueue */
int work_color; /* L: current color */
int flush_color;/* L: flushing color */
+ int refcnt; /* L: reference count */
int nr_in_flight[WORK_NR_COLORS];
/* L: nr of in_flight works */
int nr_active; /* L: nr of active works */
@@ -186,6 +187,15 @@ struct pool_workqueue {
struct list_headdelayed_works; /* L: delayed works */
struct list_headpwqs_node; /* R: node on wq-pwqs */
struct list_headmayday_node;/* W: node on wq-maydays */
+
+ /*
+* Release of unbound pwq is punted to system_wq. See put_pwq()
+* and pwq_unbound_release_workfn() for details. pool_workqueue
+* itself is also sched-RCU protected so that the first pwq can be
+* determined without grabbing workqueue_lock.
+*/
+ struct work_struct unbound_release_work;
+ struct rcu_head rcu;
} __aligned(1 WORK_STRUCT_FLAG_BITS);
/*
@@ -936,6 +946,45 @@ static void move_linked_works(struct work_struct *work,
struct list_head *head,
*nextp = n;
}
+/**
+ * get_pwq - get an extra reference on the specified pool_workqueue
+ * @pwq: pool_workqueue to get
+ *
+ * Obtain an extra reference on @pwq. The caller should guarantee that
+ * @pwq has positive refcnt and be holding the matching pool-lock.
+ */
+static void get_pwq(struct pool_workqueue *pwq)
+{
+ lockdep_assert_held(pwq-pool-lock);
+ WARN_ON_ONCE(pwq-refcnt = 0);
+ pwq-refcnt++;
+}
+
+/**
+ * put_pwq - put a pool_workqueue reference
+ * @pwq: pool_workqueue to put
+ *
+ * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
+ * destruction. The caller should be holding the matching pool-lock.
+ */
+static void put_pwq(struct pool_workqueue *pwq)
+{
+ lockdep_assert_held(pwq-pool-lock);
+ if (likely(--pwq-refcnt))
+ return;
+ if (WARN_ON_ONCE(!(pwq-wq-flags WQ_UNBOUND)))
+ return;
+ /*
+* @pwq can't be released under pool-lock, bounce to
+* pwq_unbound_release_workfn(). This never recurses on the same
+* pool-lock as this path is taken only for unbound workqueues and
+* the release work item is scheduled on a per-cpu workqueue. To
+* avoid lockdep warning, unbound pool-locks are given lockdep
+* subclass of 1 in get_unbound_pool().
+*/
+ schedule_work(pwq-unbound_release_work);
+}
+
static void pwq_activate_delayed_work(struct work_struct *work)
{
struct pool_workqueue *pwq = get_work_pwq(work);
@@ -967,9 +1016,9 @@ static void pwq_activate_first_delayed(struct
pool_workqueue *pwq)
*/
static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
{
- /* ignore uncolored works */
+ /* uncolored work items don't participate in flushing or nr_active */
if (color == WORK_NO_COLOR)
- return;
+ goto out_put;
pwq-nr_in_flight[color]--;
@@ -982,11 +1031,11 @@ static void pwq_dec_nr_in_flight(struct pool_workqueue
*pwq, int color)
/* is flush in progress and are we at the flushing tip? */
if (likely(pwq-flush_color != color))
- return;
+ goto out_put;
/* are there still in-flight works? */
if (pwq-nr_in_flight[color])
-
