On 4 August 2026 11:19:24 BST, Tvrtko Ursulin <[email protected]>
wrote:
>For use cases such as the DRM scheduler submitting work to the GPU on
>behalf of low latency userspace applications, where latter have sufficient
>privileges to have had successfully obtained realtime Vulkan global
>priority, competing with random background CPU load can create large
>latency spikes which gets in the way of a smooth user experience.
>
>For these situations the existing WQ_HIGHPRI does not bring a noticeable
>improvement and a stronger hint is needed.
>
>Lets add WQ_RTPRI which creates workers with a SCHED_FIFO scheduling class
>to improve this.
>
>We use a minimum priority level since we only care about winning the
>contest against normal background CPU load.
>
>Signed-off-by: Tvrtko Ursulin <[email protected]>
>Cc: Boris Brezillon <[email protected]>
>Cc: Bradley Morgan <[email protected]>
>Cc: Chia-I Wu <[email protected]>
>Cc: Liviu Dudau <[email protected]>
>Cc: Matthew Brost <[email protected]>
>Cc: Steven Price <[email protected]>
>Cc: Tejun Heo <[email protected]>
>---
>v2:
> * Limit WQ_RTPRI to unbound workqueues and make it have strict CPU
> affinitity. (Tejun)
> * Fixed commit message typos. (AI)
> * Fixed sysfs handling, max_active setting and user modified nice
> application. (AI)
>
>v3:
> * Fix worker->pool null pointer dereference race by moving the
> global decrement to detach_dying_workers().
> * Rebase for upstream changes.
>
>v4:
> * Fixed onion unwind.
> * Moved affinity setting to default attributes.
>
>v5:
> * Dropped global and local limits.
> * Documented in workqueue.rst.
> * Added NR_WQ_ATTRIBUTES.
> * Reverted BH handling changes.
>---
> Documentation/core-api/workqueue.rst | 5 ++
> include/linux/workqueue.h | 23 ++++++--
> kernel/workqueue.c | 80 ++++++++++++++++++++++------
> 3 files changed, 86 insertions(+), 22 deletions(-)
>
>diff --git a/Documentation/core-api/workqueue.rst
>b/Documentation/core-api/workqueue.rst
>index 411e1b28b8de..6b6179e0a38a 100644
>--- a/Documentation/core-api/workqueue.rst
>+++ b/Documentation/core-api/workqueue.rst
>@@ -225,6 +225,11 @@ resources, scheduled and executed.
> each other. Each maintains its separate pool of workers and
> implements concurrency management among its workers.
>
>+``WQ_RTPRI``
>+ Real time priority workqueues must be created as unbound and have the strict
>+ CPU affinity set. Their worker threads use the FIFO scheduling policy with
>+ the lowest priority.
>+
> ``WQ_CPU_INTENSIVE``
> Work items of a CPU intensive wq do not contribute to the
> concurrency level. In other words, runnable CPU intensive
>diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
>index a283766a192a..e9ab53568e0c 100644
>--- a/include/linux/workqueue.h
>+++ b/include/linux/workqueue.h
>@@ -140,6 +140,13 @@ enum wq_affn_scope {
> WQ_AFFN_NR_TYPES,
> };
>
>+enum wq_priority {
>+ WQ_PRIO_NORMAL = 0,
>+ WQ_PRIO_HIGH = 1,
>+ WQ_PRIO_RT = 2,
>+ NUM_WQ_PRIO, /* Keep last */
>+};
>+
> /**
> * struct workqueue_attrs - A struct for workqueue attributes.
> *
>@@ -147,7 +154,12 @@ enum wq_affn_scope {
> */
> struct workqueue_attrs {
> /**
>- * @nice: nice level
>+ * @prio: priority level
>+ */
>+ enum wq_priority prio;
>+
>+ /**
>+ * @nice: nice level for WQ_PRIO_HIGH
> */
> int nice;
>
>@@ -374,8 +386,9 @@ enum wq_flags {
> WQ_FREEZABLE = 1 << 2, /* freeze during suspend */
> WQ_MEM_RECLAIM = 1 << 3, /* may be used for memory reclaim */
> WQ_HIGHPRI = 1 << 4, /* high priority */
>- WQ_CPU_INTENSIVE = 1 << 5, /* cpu intensive workqueue */
>- WQ_SYSFS = 1 << 6, /* visible in sysfs, see
>workqueue_sysfs_register() */
>+ WQ_RTPRI = 1 << 5, /* real-time priority, valid only
>with WQ_UNBOUND */
>+ WQ_CPU_INTENSIVE = 1 << 6, /* cpu intensive workqueue */
>+ WQ_SYSFS = 1 << 7, /* visible in sysfs, see
>workqueue_sysfs_register() */
>
> /*
> * Per-cpu workqueues are generally preferred because they tend to
>@@ -402,8 +415,8 @@ enum wq_flags {
> *
> * http://thread.gmane.org/gmane.linux.kernel/1480396
> */
I feel we could clean this link up.
The link doesn't even resolve, smh man. + Its http and all of that nonsense
>- WQ_POWER_EFFICIENT = 1 << 7,
>- WQ_PERCPU = 1 << 8, /* bound to a specific cpu */
>+ WQ_POWER_EFFICIENT = 1 << 8,
>+ WQ_PERCPU = 1 << 9, /* bound to a specific cpu */
>
> __WQ_DESTROYING = 1 << 15, /* internal: workqueue is destroying
> */
> __WQ_DRAINING = 1 << 16, /* internal: workqueue is draining */
>diff --git a/kernel/workqueue.c b/kernel/workqueue.c
>index 78068ae8f28a..6b60380a8742 100644
>--- a/kernel/workqueue.c
>+++ b/kernel/workqueue.c
>@@ -105,6 +105,7 @@ enum work_cancel_flags {
>
> enum wq_internal_consts {
> NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
>+ NR_WQ_ATTRIBUTES = NUM_WQ_PRIO, /* # of attributes */
>
> UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
> BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
>@@ -509,10 +510,10 @@ static DEFINE_IDR(worker_pool_idr); /* PR: idr of
>all pools */
> static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
>
> /* I: attributes used when instantiating standard unbound pools on demand
> */
>-static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
>+static struct workqueue_attrs *unbound_std_wq_attrs[NR_WQ_ATTRIBUTES];
>
> /* I: attributes used when instantiating ordered pools on demand */
>-static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
>+static struct workqueue_attrs *ordered_wq_attrs[NR_WQ_ATTRIBUTES];
>
> /*
> * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
>@@ -2871,7 +2872,11 @@ static struct worker *create_worker(struct worker_pool
>*pool)
> goto fail;
> }
>
>- set_user_nice(worker->task, pool->attrs->nice);
>+ if (pool->attrs->prio == WQ_PRIO_RT)
>+ sched_set_fifo_low(worker->task);
>+ else
>+ set_user_nice(worker->task, pool->attrs->nice);
>+
> kthread_bind_mask(worker->task, pool_allowed_cpus(pool));
> }
>
>@@ -4780,6 +4785,7 @@ struct workqueue_attrs
>*alloc_workqueue_attrs_noprof(void)
> static void copy_workqueue_attrs(struct workqueue_attrs *to,
> const struct workqueue_attrs *from)
> {
>+ to->prio = from->prio;
> to->nice = from->nice;
> cpumask_copy(to->cpumask, from->cpumask);
> cpumask_copy(to->__pod_cpumask, from->__pod_cpumask);
>@@ -4811,6 +4817,7 @@ static u32 wqattrs_hash(const struct workqueue_attrs
>*attrs)
> {
> u32 hash = 0;
>
>+ hash = jhash_1word(attrs->prio, hash);
> hash = jhash_1word(attrs->nice, hash);
> hash = jhash_1word(attrs->affn_strict, hash);
> hash = jhash(cpumask_bits(attrs->__pod_cpumask),
>@@ -4825,6 +4832,8 @@ static u32 wqattrs_hash(const struct workqueue_attrs
>*attrs)
> static bool wqattrs_equal(const struct workqueue_attrs *a,
> const struct workqueue_attrs *b)
> {
>+ if (a->prio != b->prio)
>+ return false;
> if (a->nice != b->nice)
> return false;
> if (a->affn_strict != b->affn_strict)
>@@ -5601,11 +5610,17 @@ static void unbound_wq_update_pwq(struct
>workqueue_struct *wq, int cpu)
>
> static int alloc_and_link_pwqs(struct workqueue_struct *wq)
> {
>- bool highpri = wq->flags & WQ_HIGHPRI;
>- int cpu, ret;
>+ int prio, cpu, ret;
>
> lockdep_assert_held(&wq_pool_mutex);
>
>+ if (wq->flags & WQ_RTPRI)
>+ prio = WQ_PRIO_RT;
>+ else if (wq->flags & WQ_HIGHPRI)
>+ prio = WQ_PRIO_HIGH;
>+ else
>+ prio = WQ_PRIO_NORMAL;
>+
> wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);
> if (!wq->cpu_pwq)
> goto enomem;
>@@ -5622,7 +5637,7 @@ static int alloc_and_link_pwqs(struct workqueue_struct
>*wq)
> struct pool_workqueue **pwq_p;
> struct worker_pool *pool;
>
>- pool = &(per_cpu_ptr(pools, cpu)[highpri]);
>+ pool = &(per_cpu_ptr(pools, cpu)[prio]);
> pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu);
>
> *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,
>@@ -5642,14 +5657,14 @@ static int alloc_and_link_pwqs(struct workqueue_struct
>*wq)
> if (wq->flags & __WQ_ORDERED) {
> struct pool_workqueue *dfl_pwq;
>
>- ret = apply_workqueue_attrs_locked(wq,
>ordered_wq_attrs[highpri]);
>+ ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[prio]);
> /* there should only be single pwq for ordering guarantee */
> dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
> WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
> wq->pwqs.prev != &dfl_pwq->pwqs_node),
> "ordering guarantee broken for workqueue %s\n", wq->name);
> } else {
>- ret = apply_workqueue_attrs_locked(wq,
>unbound_std_wq_attrs[highpri]);
>+ ret = apply_workqueue_attrs_locked(wq,
>unbound_std_wq_attrs[prio]);
> }
>
> if (ret)
>@@ -5814,6 +5829,12 @@ static struct workqueue_struct *__alloc_workqueue(const
>char *fmt,
> return NULL;
> }
>
>+ if (flags & WQ_RTPRI) {
>+ if (WARN_ON_ONCE((flags & (WQ_HIGHPRI | WQ_UNBOUND)) !=
>+ WQ_UNBOUND))
>+ return NULL;
>+ }
>+
> /* see the comment above the definition of WQ_POWER_EFFICIENT */
> if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
> flags = (flags & ~WQ_PERCPU) | WQ_UNBOUND;
>@@ -7302,7 +7323,11 @@ static ssize_t wq_nice_show(struct device *dev, struct
>device_attribute *attr,
> int written;
>
> mutex_lock(&wq->mutex);
>- written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
>+ if (wq->unbound_attrs->prio != WQ_PRIO_RT)
>+ written = scnprintf(buf, PAGE_SIZE, "%d\n",
>+ wq->unbound_attrs->nice);
>+ else
>+ written = -EINVAL;
> mutex_unlock(&wq->mutex);
>
> return written;
>@@ -7328,13 +7353,20 @@ static ssize_t wq_nice_store(struct device *dev,
>struct device_attribute *attr,
> {
> struct workqueue_struct *wq = dev_to_wq(dev);
> struct workqueue_attrs *attrs;
>- int ret = -ENOMEM;
>+ int ret;
>
> mutex_lock(&wq_pool_mutex);
>
> attrs = wq_sysfs_prep_attrs(wq);
>- if (!attrs)
>+ if (!attrs) {
>+ ret = -ENOMEM;
> goto out_unlock;
>+ }
>+
>+ if (attrs->prio == WQ_PRIO_RT) {
>+ ret = -EINVAL;
>+ goto out_unlock;
>+ }
>
> if (sscanf(buf, "%d", &attrs->nice) == 1 &&
> attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
>@@ -7942,12 +7974,13 @@ static void __init restrict_unbound_cpumask(const char
>*name, const struct cpuma
> cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, mask);
> }
>
>-static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu,
>int nice)
>+static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu,
>enum wq_priority prio, int nice)
> {
> BUG_ON(init_worker_pool(pool));
> pool->cpu = cpu;
> cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
> cpumask_copy(pool->attrs->__pod_cpumask, cpumask_of(cpu));
>+ pool->attrs->prio = prio;
> pool->attrs->nice = nice;
> pool->attrs->affn_strict = true;
> pool->node = cpu_to_node(cpu);
>@@ -7971,7 +8004,8 @@ static void __init init_cpu_worker_pool(struct
>worker_pool *pool, int cpu, int n
> void __init workqueue_init_early(void)
> {
> struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM];
>- int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
>+ int std_prio[NR_WQ_ATTRIBUTES] = { 0, WQ_PRIO_HIGH, WQ_PRIO_RT };
>+ int std_nice[NR_WQ_ATTRIBUTES] = { 0, HIGHPRI_NICE_LEVEL, 0 };
> void (*irq_work_fns[NR_STD_WORKER_POOLS])(struct irq_work *) =
> { bh_pool_kick_normal, bh_pool_kick_highpri };
> int i, cpu;
>@@ -8023,23 +8057,34 @@ void __init workqueue_init_early(void)
>
> i = 0;
> for_each_bh_worker_pool(pool, cpu) {
>- init_cpu_worker_pool(pool, cpu, std_nice[i]);
>+ init_cpu_worker_pool(pool, cpu, std_prio[i],
>std_nice[i]);
> pool->flags |= POOL_BH;
> init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);
> i++;
> }
>
> i = 0;
>- for_each_cpu_worker_pool(pool, cpu)
>- init_cpu_worker_pool(pool, cpu, std_nice[i++]);
>+ for_each_cpu_worker_pool(pool, cpu) {
>+ init_cpu_worker_pool(pool, cpu, std_prio[i],
>std_nice[i]);
>+ i++;
>+ }
> }
>
> /* create default unbound and ordered wq attrs */
>- for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
>+ for (i = 0; i < NR_WQ_ATTRIBUTES; i++) {
> struct workqueue_attrs *attrs;
>
> BUG_ON(!(attrs = alloc_workqueue_attrs()));
>+ attrs->prio = std_prio[i];
> attrs->nice = std_nice[i];
>+ if (i == WQ_PRIO_RT) {
>+ /*
>+ * RT workqueues have strict CPU affinity for low
>+ * latency execution.
>+ */
>+ attrs->affn_scope = WQ_AFFN_CPU;
>+ attrs->affn_strict = true;
>+ }
> unbound_std_wq_attrs[i] = attrs;
>
> /*
>@@ -8047,6 +8092,7 @@ void __init workqueue_init_early(void)
> * guaranteed by max_active which is enforced by pwqs.
> */
> BUG_ON(!(attrs = alloc_workqueue_attrs()));
>+ attrs->prio = std_prio[i];
> attrs->nice = std_nice[i];
> attrs->ordered = true;
> ordered_wq_attrs[i] = attrs;
>
Thanks!
