[tip: sched/core] rseq: Optimize rseq_update_cpu_id()
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 60af388d23889636011488c42763876bcdda3eab
Gitweb:
https://git.kernel.org/tip/60af388d23889636011488c42763876bcdda3eab
Author:Eric Dumazet
AuthorDate:Tue, 13 Apr 2021 13:33:50 -07:00
Committer: Peter Zijlstra
CommitterDate: Wed, 14 Apr 2021 18:04:09 +02:00
rseq: Optimize rseq_update_cpu_id()
Two put_user() in rseq_update_cpu_id() are replaced
by a pair of unsafe_put_user() with appropriate surroundings.
This removes one stac/clac pair on x86 in fast path.
Signed-off-by: Eric Dumazet
Signed-off-by: Peter Zijlstra (Intel)
Reviewed-by: Mathieu Desnoyers
Link: https://lkml.kernel.org/r/20210413203352.71350-2-eric.duma...@gmail.com
---
kernel/rseq.c | 15 +++
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/kernel/rseq.c b/kernel/rseq.c
index a4f86a9..f020f18 100644
--- a/kernel/rseq.c
+++ b/kernel/rseq.c
@@ -84,13 +84,20 @@
static int rseq_update_cpu_id(struct task_struct *t)
{
u32 cpu_id = raw_smp_processor_id();
+ struct rseq __user *rseq = t->rseq;
- if (put_user(cpu_id, >rseq->cpu_id_start))
- return -EFAULT;
- if (put_user(cpu_id, >rseq->cpu_id))
- return -EFAULT;
+ if (!user_write_access_begin(rseq, sizeof(*rseq)))
+ goto efault;
+ unsafe_put_user(cpu_id, >cpu_id_start, efault_end);
+ unsafe_put_user(cpu_id, >cpu_id, efault_end);
+ user_write_access_end();
trace_rseq_update(t);
return 0;
+
+efault_end:
+ user_write_access_end();
+efault:
+ return -EFAULT;
}
static int rseq_reset_rseq_cpu_id(struct task_struct *t)
[tip: sched/core] rseq: Remove redundant access_ok()
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 0ed96051531ecc6965f6456d25b19b9b6bdb5c28
Gitweb:
https://git.kernel.org/tip/0ed96051531ecc6965f6456d25b19b9b6bdb5c28
Author:Eric Dumazet
AuthorDate:Tue, 13 Apr 2021 13:33:51 -07:00
Committer: Peter Zijlstra
CommitterDate: Wed, 14 Apr 2021 18:04:09 +02:00
rseq: Remove redundant access_ok()
After commit 8f2817701492 ("rseq: Use get_user/put_user rather
than __get_user/__put_user") we no longer need
an access_ok() call from __rseq_handle_notify_resume()
Mathieu pointed out the same cleanup can be done
in rseq_syscall().
Signed-off-by: Eric Dumazet
Signed-off-by: Peter Zijlstra (Intel)
Reviewed-by: Mathieu Desnoyers
Link: https://lkml.kernel.org/r/20210413203352.71350-3-eric.duma...@gmail.com
---
kernel/rseq.c | 5 +
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/kernel/rseq.c b/kernel/rseq.c
index f020f18..cfe01ab 100644
--- a/kernel/rseq.c
+++ b/kernel/rseq.c
@@ -273,8 +273,6 @@ void __rseq_handle_notify_resume(struct ksignal *ksig,
struct pt_regs *regs)
if (unlikely(t->flags & PF_EXITING))
return;
- if (unlikely(!access_ok(t->rseq, sizeof(*t->rseq
- goto error;
ret = rseq_ip_fixup(regs);
if (unlikely(ret < 0))
goto error;
@@ -301,8 +299,7 @@ void rseq_syscall(struct pt_regs *regs)
if (!t->rseq)
return;
- if (!access_ok(t->rseq, sizeof(*t->rseq)) ||
- rseq_get_rseq_cs(t, _cs) || in_rseq_cs(ip, _cs))
+ if (rseq_get_rseq_cs(t, _cs) || in_rseq_cs(ip, _cs))
force_sig(SIGSEGV);
}
[tip: sched/core] rseq: Optimise rseq_get_rseq_cs() and clear_rseq_cs()
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 5e0ccd4a3b01c5a71732a13186ca110a138516ea
Gitweb:
https://git.kernel.org/tip/5e0ccd4a3b01c5a71732a13186ca110a138516ea
Author:Eric Dumazet
AuthorDate:Tue, 13 Apr 2021 13:33:52 -07:00
Committer: Peter Zijlstra
CommitterDate: Wed, 14 Apr 2021 18:04:09 +02:00
rseq: Optimise rseq_get_rseq_cs() and clear_rseq_cs()
Commit ec9c82e03a74 ("rseq: uapi: Declare rseq_cs field as union,
update includes") added regressions for our servers.
Using copy_from_user() and clear_user() for 64bit values
is suboptimal.
We can use faster put_user() and get_user() on 64bit arches.
Signed-off-by: Eric Dumazet
Signed-off-by: Peter Zijlstra (Intel)
Reviewed-by: Mathieu Desnoyers
Link: https://lkml.kernel.org/r/20210413203352.71350-4-eric.duma...@gmail.com
---
kernel/rseq.c | 9 +
1 file changed, 9 insertions(+)
diff --git a/kernel/rseq.c b/kernel/rseq.c
index cfe01ab..35f7bd0 100644
--- a/kernel/rseq.c
+++ b/kernel/rseq.c
@@ -127,8 +127,13 @@ static int rseq_get_rseq_cs(struct task_struct *t, struct
rseq_cs *rseq_cs)
u32 sig;
int ret;
+#ifdef CONFIG_64BIT
+ if (get_user(ptr, >rseq->rseq_cs.ptr64))
+ return -EFAULT;
+#else
if (copy_from_user(, >rseq->rseq_cs.ptr64, sizeof(ptr)))
return -EFAULT;
+#endif
if (!ptr) {
memset(rseq_cs, 0, sizeof(*rseq_cs));
return 0;
@@ -211,9 +216,13 @@ static int clear_rseq_cs(struct task_struct *t)
*
* Set rseq_cs to NULL.
*/
+#ifdef CONFIG_64BIT
+ return put_user(0UL, >rseq->rseq_cs.ptr64);
+#else
if (clear_user(>rseq->rseq_cs.ptr64, sizeof(t->rseq->rseq_cs.ptr64)))
return -EFAULT;
return 0;
+#endif
}
/*
[tip: timers/urgent] hrtimer: Annotate lockless access to timer->base
The following commit has been merged into the timers/urgent branch of tip:
Commit-ID: ff229eee3d897f52bd001c841f2d3cce8853ecdc
Gitweb:
https://git.kernel.org/tip/ff229eee3d897f52bd001c841f2d3cce8853ecdc
Author:Eric Dumazet
AuthorDate:Tue, 08 Oct 2019 10:32:04 -07:00
Committer: Thomas Gleixner
CommitterDate: Mon, 14 Oct 2019 15:51:49 +02:00
hrtimer: Annotate lockless access to timer->base
Followup to commit dd2261ed45aa ("hrtimer: Protect lockless access
to timer->base")
lock_hrtimer_base() fetches timer->base without lock exclusion.
Compiler is allowed to read timer->base twice (even if considered dumb)
which could end up trying to lock migration_base and return
_base.
base = timer->base;
if (likely(base != _base)) {
/* compiler reads timer->base again, and now (base == _base)
raw_spin_lock_irqsave(>cpu_base->lock, *flags);
if (likely(base == timer->base))
return base; /* == _base ! */
Similarly the write sides must use WRITE_ONCE() to avoid store tearing.
Signed-off-by: Eric Dumazet
Signed-off-by: Thomas Gleixner
Link: https://lkml.kernel.org/r/20191008173204.180879-1-eduma...@google.com
---
kernel/time/hrtimer.c | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 0d4dc24..6560553 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -164,7 +164,7 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct
hrtimer *timer,
struct hrtimer_clock_base *base;
for (;;) {
- base = timer->base;
+ base = READ_ONCE(timer->base);
if (likely(base != _base)) {
raw_spin_lock_irqsave(>cpu_base->lock, *flags);
if (likely(base == timer->base))
@@ -244,7 +244,7 @@ again:
return base;
/* See the comment in lock_hrtimer_base() */
- timer->base = _base;
+ WRITE_ONCE(timer->base, _base);
raw_spin_unlock(>cpu_base->lock);
raw_spin_lock(_base->cpu_base->lock);
@@ -253,10 +253,10 @@ again:
raw_spin_unlock(_base->cpu_base->lock);
raw_spin_lock(>cpu_base->lock);
new_cpu_base = this_cpu_base;
- timer->base = base;
+ WRITE_ONCE(timer->base, base);
goto again;
}
- timer->base = new_base;
+ WRITE_ONCE(timer->base, new_base);
} else {
if (new_cpu_base != this_cpu_base &&
hrtimer_check_target(timer, new_base)) {
