advance_sched() advances exactly one entry per hrtimer expiry. When the
operational schedule falls behind - the timer was delayed, the CPU was
starved, or the reference clock stepped forward - every elapsed entry
is replayed back to back from hrtimer context with current_entry_lock
held, and each replay rearms the timer with an expiry in the past. Once
the backlog is large enough the CPU never leaves timer processing and
RCU stalls follow. syzbot triggers this with schedules whose intervals
are shorter than the cost of servicing one expiry, so the backlog only
ever grows.
Skip complete cycles arithmetically and walk at most one cycle of
entries to land on the entry covering the current time. Gate close
times and budgets are still only computed for the entry landed on. An
admin schedule crossed by the jump is picked up by the existing
should_change_schedules() check on the recomputed end time. The walk is
capped at twice the entry count as a safeguard against degenerate
intervals; leftover backlog is then handled by the next expiry as
today.
Fixes: 5a781ccbd19e ("tc: Add support for configuring the taprio scheduler")
Signed-off-by: Junjie Cao <[email protected]>
---
net/sched/sch_taprio.c | 56 +++++++++++++++++++++++++++++++++++++++---
1 file changed, 53 insertions(+), 3 deletions(-)
diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c
index 299234a5f0fe..f3f90c5d2dca 100644
--- a/net/sched/sch_taprio.c
+++ b/net/sched/sch_taprio.c
@@ -915,6 +915,51 @@ static bool should_change_schedules(const struct
sched_gate_list *admin,
return false;
}
+/* The operational schedule fell behind, e.g. because the timer was delayed
+ * or the reference clock stepped forward. Advancing one entry per timer
+ * expiry would replay the whole backlog from hrtimer context, so skip
+ * complete cycles arithmetically and walk the remaining entries to land on
+ * the entry covering the current time.
+ */
+static void taprio_catch_up(struct sched_gate_list *oper,
+ struct sched_entry **next, ktime_t *next_start,
+ ktime_t *end_time, ktime_t now)
+{
+ int budget = 2 * oper->num_entries + 1;
+ struct sched_entry *entry = *next;
+ ktime_t start = *next_start;
+ ktime_t end = *end_time;
+ s64 behind = ktime_sub(now, end);
+
+ if (oper->cycle_time > 0 && behind >= oper->cycle_time) {
+ s64 jump = div64_s64(behind, oper->cycle_time) *
oper->cycle_time;
+
+ start = ktime_add_ns(start, jump);
+ end = ktime_add_ns(end, jump);
+ oper->cycle_end_time = ktime_add_ns(oper->cycle_end_time, jump);
+ }
+
+ while (ktime_before(end, now) && --budget) {
+ if (list_is_last(&entry->list, &oper->entries) ||
+ ktime_compare(end, oper->cycle_end_time) == 0) {
+ entry = list_first_entry(&oper->entries,
+ struct sched_entry, list);
+ oper->cycle_end_time =
ktime_add_ns(oper->cycle_end_time,
+ oper->cycle_time);
+ } else {
+ entry = list_next_entry(entry, list);
+ }
+
+ start = end;
+ end = ktime_add_ns(end, entry->interval);
+ end = min_t(ktime_t, end, oper->cycle_end_time);
+ }
+
+ *next = entry;
+ *next_start = start;
+ *end_time = end;
+}
+
static enum hrtimer_restart advance_sched(struct hrtimer *timer)
{
struct taprio_sched *q = container_of(timer, struct taprio_sched,
@@ -924,7 +969,7 @@ static enum hrtimer_restart advance_sched(struct hrtimer
*timer)
int num_tc = netdev_get_num_tc(dev);
struct sched_entry *entry, *next;
struct Qdisc *sch = q->root;
- ktime_t end_time;
+ ktime_t end_time, next_start, now;
int tc;
spin_lock(&q->current_entry_lock);
@@ -960,14 +1005,19 @@ static enum hrtimer_restart advance_sched(struct hrtimer
*timer)
next = list_next_entry(entry, list);
}
- end_time = ktime_add_ns(entry->end_time, next->interval);
+ next_start = entry->end_time;
+ end_time = ktime_add_ns(next_start, next->interval);
end_time = min_t(ktime_t, end_time, oper->cycle_end_time);
+ now = taprio_get_time(q);
+ if (unlikely(ktime_before(end_time, now)))
+ taprio_catch_up(oper, &next, &next_start, &end_time, now);
+
for (tc = 0; tc < num_tc; tc++) {
if (next->gate_duration[tc] == oper->cycle_time)
next->gate_close_time[tc] = KTIME_MAX;
else
- next->gate_close_time[tc] =
ktime_add_ns(entry->end_time,
+ next->gate_close_time[tc] = ktime_add_ns(next_start,
next->gate_duration[tc]);
}
--
2.43.0
