Re: [RFC PATCH 23/27] powerpc/64: system call implement the bulk of the logic in C
On Sun, Sep 15, 2019 at 11:28:09AM +1000, Nicholas Piggin wrote:
> System call entry and particularly exit code is beyond the limit of what
> is reasonable to implement in asm.
>
> This conversion moves all conditional branches out of the asm code,
> except for the case that all GPRs should be restored at exit.
>
> Null syscall test is about 5% faster after this patch, because the exit
> work is handled under local_irq_disable, and the hard mask and pending
> interrupt replay is handled after that, which avoids games with MSR.
>
> Signed-off-by: Nicholas Piggin
>
> v3:
> - Fix !KUAP build [mpe]
> - Fix BookE build/boot [mpe]
> - Don't trace irqs with MSR[RI]=0
> - Don't allow syscall_exit_prepare to be ftraced, because function
> graph tracing which traces exits barfs after the IRQ state is
> prepared for kernel exit.
> - Fix BE syscall table to use normal function descriptors now that they
> are called from C.
> - Comment syscall_exit_prepare.
...
> -#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
> -BEGIN_FW_FTR_SECTION
> - /* see if there are any DTL entries to process */
> - ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
> - ld r11,PACA_DTL_RIDX(r13) /* get log read index */
> - addir10,r10,LPPACA_DTLIDX
> - LDX_BE r10,0,r10 /* get log write index */
> - cmpdr11,r10
> - beq+33f
> - bl accumulate_stolen_time
> - REST_GPR(0,r1)
> - REST_4GPRS(3,r1)
> - REST_2GPRS(7,r1)
> - addir9,r1,STACK_FRAME_OVERHEAD
> -33:
> -END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
> -#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
...
> diff --git a/arch/powerpc/kernel/syscall_64.c
> b/arch/powerpc/kernel/syscall_64.c
> new file mode 100644
> index ..1d2529824588
> --- /dev/null
> +++ b/arch/powerpc/kernel/syscall_64.c
> @@ -0,0 +1,195 @@
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +
> +extern void __noreturn tabort_syscall(void);
> +
> +typedef long (*syscall_fn)(long, long, long, long, long, long);
> +
> +long system_call_exception(long r3, long r4, long r5, long r6, long r7, long
> r8, unsigned long r0, struct pt_regs *regs)
> +{
> + unsigned long ti_flags;
> + syscall_fn f;
> +
> + BUG_ON(!(regs->msr & MSR_PR));
> +
> + if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
> + unlikely(regs->msr & MSR_TS_T))
> + tabort_syscall();
> +
> + account_cpu_user_entry();
> +
> +#ifdef CONFIG_PPC_SPLPAR
> + if (IS_ENABLED(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) &&
> + firmware_has_feature(FW_FEATURE_SPLPAR)) {
> + struct lppaca *lp = get_lppaca();
> +
> + if (unlikely(local_paca->dtl_ridx != lp->dtl_idx))
sparse complains about this, and in time.c this it converted like this:
if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx)))
The removed code has a LDX_BE there so there should be some conversion
involved, right?
Thanks
Michal
Re: [RFC PATCH 23/27] powerpc/64: system call implement the bulk of the logic in C
Michal Suchánek's on September 23, 2019 10:47 pm:
> On Sun, Sep 15, 2019 at 11:28:09AM +1000, Nicholas Piggin wrote:
>> System call entry and particularly exit code is beyond the limit of what
>> is reasonable to implement in asm.
>>
>> This conversion moves all conditional branches out of the asm code,
>> except for the case that all GPRs should be restored at exit.
>>
>> Null syscall test is about 5% faster after this patch, because the exit
>> work is handled under local_irq_disable, and the hard mask and pending
>> interrupt replay is handled after that, which avoids games with MSR.
>>
>> Signed-off-by: Nicholas Piggin
>>
>> v3:
>> - Fix !KUAP build [mpe]
>> - Fix BookE build/boot [mpe]
>> - Don't trace irqs with MSR[RI]=0
>> - Don't allow syscall_exit_prepare to be ftraced, because function
>> graph tracing which traces exits barfs after the IRQ state is
>> prepared for kernel exit.
>> - Fix BE syscall table to use normal function descriptors now that they
>> are called from C.
>> - Comment syscall_exit_prepare.
> ...
>> -#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
>> -BEGIN_FW_FTR_SECTION
>> -/* see if there are any DTL entries to process */
>> -ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
>> -ld r11,PACA_DTL_RIDX(r13) /* get log read index */
>> -addir10,r10,LPPACA_DTLIDX
>> -LDX_BE r10,0,r10 /* get log write index */
>> -cmpdr11,r10
>> -beq+33f
>> -bl accumulate_stolen_time
>> -REST_GPR(0,r1)
>> -REST_4GPRS(3,r1)
>> -REST_2GPRS(7,r1)
>> -addir9,r1,STACK_FRAME_OVERHEAD
>> -33:
>> -END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
>> -#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
> ...
>> diff --git a/arch/powerpc/kernel/syscall_64.c
>> b/arch/powerpc/kernel/syscall_64.c
>> new file mode 100644
>> index ..1d2529824588
>> --- /dev/null
>> +++ b/arch/powerpc/kernel/syscall_64.c
>> @@ -0,0 +1,195 @@
>> +#include
>> +#include
>> +#include
>> +#include
>> +#include
>> +#include
>> +#include
>> +#include
>> +#include
>> +#include
>> +#include
>> +#include
>> +
>> +extern void __noreturn tabort_syscall(void);
>> +
>> +typedef long (*syscall_fn)(long, long, long, long, long, long);
>> +
>> +long system_call_exception(long r3, long r4, long r5, long r6, long r7,
>> long r8, unsigned long r0, struct pt_regs *regs)
>> +{
>> +unsigned long ti_flags;
>> +syscall_fn f;
>> +
>> +BUG_ON(!(regs->msr & MSR_PR));
>> +
>> +if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
>> +unlikely(regs->msr & MSR_TS_T))
>> +tabort_syscall();
>> +
>> +account_cpu_user_entry();
>> +
>> +#ifdef CONFIG_PPC_SPLPAR
>> +if (IS_ENABLED(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) &&
>> +firmware_has_feature(FW_FEATURE_SPLPAR)) {
>> +struct lppaca *lp = get_lppaca();
>> +
>> +if (unlikely(local_paca->dtl_ridx != lp->dtl_idx))
>
> sparse complains about this, and in time.c this it converted like this:
> if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx)))
> The removed code has a LDX_BE there so there should be some conversion
> involved, right?
Ah yes, thanks good catch.
Thanks,
Nick
[RFC PATCH 23/27] powerpc/64: system call implement the bulk of the logic in C
System call entry and particularly exit code is beyond the limit of what
is reasonable to implement in asm.
This conversion moves all conditional branches out of the asm code,
except for the case that all GPRs should be restored at exit.
Null syscall test is about 5% faster after this patch, because the exit
work is handled under local_irq_disable, and the hard mask and pending
interrupt replay is handled after that, which avoids games with MSR.
Signed-off-by: Nicholas Piggin
v3:
- Fix !KUAP build [mpe]
- Fix BookE build/boot [mpe]
- Don't trace irqs with MSR[RI]=0
- Don't allow syscall_exit_prepare to be ftraced, because function
graph tracing which traces exits barfs after the IRQ state is
prepared for kernel exit.
- Fix BE syscall table to use normal function descriptors now that they
are called from C.
- Comment syscall_exit_prepare.
---
arch/powerpc/include/asm/asm-prototypes.h | 11 -
.../powerpc/include/asm/book3s/64/kup-radix.h | 14 +-
arch/powerpc/include/asm/cputime.h| 24 ++
arch/powerpc/include/asm/hw_irq.h | 4 +
arch/powerpc/include/asm/ptrace.h | 3 +
arch/powerpc/include/asm/signal.h | 3 +
arch/powerpc/include/asm/switch_to.h | 5 +
arch/powerpc/include/asm/time.h | 3 +
arch/powerpc/kernel/Makefile | 3 +-
arch/powerpc/kernel/entry_64.S| 337 +++---
arch/powerpc/kernel/signal.h | 2 -
arch/powerpc/kernel/syscall_64.c | 195 ++
arch/powerpc/kernel/systbl.S | 9 +-
13 files changed, 300 insertions(+), 313 deletions(-)
create mode 100644 arch/powerpc/kernel/syscall_64.c
diff --git a/arch/powerpc/include/asm/asm-prototypes.h
b/arch/powerpc/include/asm/asm-prototypes.h
index 49196d35e3bb..cb5e85475150 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -103,14 +103,6 @@ long sys_switch_endian(void);
notrace unsigned int __check_irq_replay(void);
void notrace restore_interrupts(void);
-/* ptrace */
-long do_syscall_trace_enter(struct pt_regs *regs);
-void do_syscall_trace_leave(struct pt_regs *regs);
-
-/* process */
-void restore_math(struct pt_regs *regs);
-void restore_tm_state(struct pt_regs *regs);
-
/* prom_init (OpenFirmware) */
unsigned long __init prom_init(unsigned long r3, unsigned long r4,
unsigned long pp,
@@ -121,9 +113,6 @@ unsigned long __init prom_init(unsigned long r3, unsigned
long r4,
void __init early_setup(unsigned long dt_ptr);
void early_setup_secondary(void);
-/* time */
-void accumulate_stolen_time(void);
-
/* misc runtime */
extern u64 __bswapdi2(u64);
extern s64 __lshrdi3(s64, int);
diff --git a/arch/powerpc/include/asm/book3s/64/kup-radix.h
b/arch/powerpc/include/asm/book3s/64/kup-radix.h
index f254de956d6a..07058edc5970 100644
--- a/arch/powerpc/include/asm/book3s/64/kup-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/kup-radix.h
@@ -3,6 +3,7 @@
#define _ASM_POWERPC_BOOK3S_64_KUP_RADIX_H
#include
+#include
#define AMR_KUAP_BLOCK_READUL(0x4000)
#define AMR_KUAP_BLOCK_WRITE UL(0x8000)
@@ -56,7 +57,14 @@
#ifdef CONFIG_PPC_KUAP
-#include
+#include
+#include
+
+static inline void kuap_check_amr(void)
+{
+ if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) &&
mmu_has_feature(MMU_FTR_RADIX_KUAP))
+ WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);
+}
/*
* We support individually allowing read or write, but we don't support nesting
@@ -101,6 +109,10 @@ static inline bool bad_kuap_fault(struct pt_regs *regs,
bool is_write)
(regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE :
AMR_KUAP_BLOCK_READ)),
"Bug: %s fault blocked by AMR!", is_write ? "Write" :
"Read");
}
+#else /* CONFIG_PPC_KUAP */
+static inline void kuap_check_amr(void)
+{
+}
#endif /* CONFIG_PPC_KUAP */
#endif /* __ASSEMBLY__ */
diff --git a/arch/powerpc/include/asm/cputime.h
b/arch/powerpc/include/asm/cputime.h
index 2431b4ada2fa..c43614cffaac 100644
--- a/arch/powerpc/include/asm/cputime.h
+++ b/arch/powerpc/include/asm/cputime.h
@@ -60,6 +60,30 @@ static inline void arch_vtime_task_switch(struct task_struct
*prev)
}
#endif
+static inline void account_cpu_user_entry(void)
+{
+ unsigned long tb = mftb();
+ struct cpu_accounting_data *acct = get_accounting(current);
+
+ acct->utime += (tb - acct->starttime_user);
+ acct->starttime = tb;
+}
+static inline void account_cpu_user_exit(void)
+{
+ unsigned long tb = mftb();
+ struct cpu_accounting_data *acct = get_accounting(current);
+
+ acct->stime += (tb - acct->starttime);
+ acct->starttime_user = tb;
+}
+
#endif /* __KERNEL__ */
+#else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
+static inline void account_cpu_user_entry(void)
+{
+}
+static inline void account_cpu_user_exit(void)
+{
+}
#endif /*
