Introduce housekeeping_update_types(), which updates the cpumask for
each specified housekeeping type atomically using an RCU pointer swap.
For each type in @type_mask the trial mask is computed as
(base & ~isol_mask), where the base depends on the type:
- Most types use the current housekeeping cpumask as base. For
types that are only set at boot this is equivalent to the boot
mask, so trial = (boot_mask & ~isol_mask).
- HK_TYPE_KERNEL_NOISE always uses cpu_possible_mask as base. Its
semantics are "all possible CPUs minus the currently-isolated set";
using the current HK mask instead would leave it stuck at its last
non-trivial value after de-isolation, breaking subsequent isolation
cycles.
HK_TYPE_KERNEL_NOISE also supports runtime first-enable: if it was not
registered at boot (no nohz_full= on the kernel command line),
housekeeping_update_types() registers it in housekeeping.flags on the
first call. All other types must already be boot-enabled.
For each type the function validates the trial mask against
cpu_online_mask, runs registered pre_validate() callbacks (which may
reject the update), swaps all RCU cpumask pointers in a single pass,
calls synchronize_rcu(), frees the old masks, and then runs apply()
callbacks.
The existing housekeeping_update() continues to update only
HK_TYPE_DOMAIN and remains the entry point for the cpuset partition
path. housekeeping_update_types() enables the partition path to also
drive the kernel-noise types (HK_TYPE_KERNEL_NOISE,
HK_TYPE_MANAGED_IRQ) through the explicit callback interface added in
the previous patch.
Signed-off-by: Jing Wu <[email protected]>
Signed-off-by: Qiliang Yuan <[email protected]>
---
include/linux/sched/isolation.h | 4 ++
kernel/sched/isolation.c | 112 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 116 insertions(+)
diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h
index f362876b3ebdf..eecbcbe802bd0 100644
--- a/include/linux/sched/isolation.h
+++ b/include/linux/sched/isolation.h
@@ -44,6 +44,8 @@ extern bool housekeeping_enabled(enum hk_type type);
extern void housekeeping_affine(struct task_struct *t, enum hk_type type);
extern bool housekeeping_test_cpu(int cpu, enum hk_type type);
extern int housekeeping_update(struct cpumask *isol_mask);
+extern int housekeeping_update_types(unsigned long type_mask,
+ struct cpumask *isol_mask);
extern void __init housekeeping_init(void);
/**
@@ -99,6 +101,8 @@ static inline bool housekeeping_test_cpu(int cpu, enum
hk_type type)
}
static inline int housekeeping_update(struct cpumask *isol_mask) { return 0; }
+static inline int housekeeping_update_types(unsigned long type_mask,
+ struct cpumask *isol_mask) { return
0; }
static inline void housekeeping_init(void) { }
static inline int housekeeping_register_cbs(enum hk_type type,
struct housekeeping_cbs *cbs) {
return 0; }
diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c
index aae4dff7fbfc8..4eca18cc5e8ce 100644
--- a/kernel/sched/isolation.c
+++ b/kernel/sched/isolation.c
@@ -249,6 +249,118 @@ int housekeeping_update(struct cpumask *isol_mask)
return 0;
}
+/**
+ * housekeeping_update_types - Update housekeeping masks for specified types
+ * @type_mask: Bitmask of housekeeping types to update
+ * @isol_mask: CPUs being added to the isolation set
+ *
+ * For each type in @type_mask that was enabled at boot, compute the
+ * trial mask as (boot mask & ~@isol_mask), validate it against
+ * @cpu_online_mask, invoke pre_validate() callbacks, swap the RCU
+ * mask pointer, and run apply() callbacks after synchronize_rcu().
+ *
+ * HK_TYPE_KERNEL_NOISE also supports runtime first-enable: when an
+ * isolated cpuset partition is created without nohz_full= at boot,
+ * cpu_possible_mask is used as the initial base and the type flag is
+ * set in housekeeping.flags on the first call.
+ *
+ * Return: 0 on success, -ENOMEM on allocation failure, -EINVAL if
+ * a trial mask has no online CPUs.
+ */
+int housekeeping_update_types(unsigned long type_mask,
+ struct cpumask *isol_mask)
+{
+ struct cpumask *trials[HK_TYPE_MAX] = {};
+ struct cpumask *old_masks[HK_TYPE_MAX] = {};
+ enum hk_type type;
+ int ret = 0;
+
+ for_each_set_bit(type, &type_mask, HK_TYPE_MAX) {
+ const struct cpumask *base;
+
+ if (type == HK_TYPE_DOMAIN_BOOT)
+ continue;
+ if (!housekeeping_enabled(type)) {
+ /*
+ * HK_TYPE_KERNEL_NOISE supports runtime first-enable
+ * for DHM isolated partitions created without
nohz_full=
+ * at boot. All other types must be boot-enabled.
+ */
+ if (type != HK_TYPE_KERNEL_NOISE)
+ continue;
+ }
+
+ /*
+ * HK_TYPE_KERNEL_NOISE always uses cpu_possible_mask as its
+ * base. Its semantics are exactly "cpu_possible minus the
+ * currently-isolated set", so the base never shrinks across
+ * successive isolation/de-isolation cycles. If we used the
+ * current HK mask instead, de-isolating all partitions would
+ * leave the mask at its last non-trivial value rather than
+ * reverting to cpu_possible, breaking subsequent isolations.
+ */
+ if (type == HK_TYPE_KERNEL_NOISE)
+ base = cpu_possible_mask;
+ else
+ base = housekeeping_cpumask(type);
+ trials[type] = kmalloc(cpumask_size(), GFP_KERNEL);
+ if (!trials[type]) {
+ ret = -ENOMEM;
+ goto err_free;
+ }
+ cpumask_andnot(trials[type], base, isol_mask);
+ if (!cpumask_intersects(trials[type], cpu_online_mask)) {
+ ret = -EINVAL;
+ goto err_free;
+ }
+ }
+
+ if (!housekeeping.flags) {
+ ret = -EINVAL;
+ goto err_free;
+ }
+
+ for_each_set_bit(type, &type_mask, HK_TYPE_MAX) {
+ if (!trials[type])
+ continue;
+ ret = housekeeping_pre_validate_cbs(type,
+ housekeeping_cpumask(type),
+ trials[type]);
+ if (ret < 0)
+ goto err_free;
+ }
+
+ for_each_set_bit(type, &type_mask, HK_TYPE_MAX) {
+ if (!trials[type])
+ continue;
+ old_masks[type] = housekeeping_cpumask_dereference(type);
+ /* First-time runtime enable: register the type now. */
+ if (!housekeeping_enabled(type))
+ WRITE_ONCE(housekeeping.flags,
+ housekeeping.flags | BIT(type));
+ rcu_assign_pointer(housekeeping.cpumasks[type], trials[type]);
+ trials[type] = NULL;
+ }
+
+ synchronize_rcu();
+
+ for_each_set_bit(type, &type_mask, HK_TYPE_MAX) {
+ if (housekeeping_cbs_table[type].nr == 0)
+ continue;
+ housekeeping_apply_cbs(type);
+ }
+
+ for_each_set_bit(type, &type_mask, HK_TYPE_MAX)
+ kfree(old_masks[type]);
+
+ return 0;
+
+err_free:
+ for_each_set_bit(type, &type_mask, HK_TYPE_MAX)
+ kfree(trials[type]);
+ return ret;
+}
+
void __init housekeeping_init(void)
{
enum hk_type type;
--
2.43.0
