Author: kib
Date: Sun Nov 18 09:49:24 2018
New Revision: 340552
URL: https://svnweb.freebsd.org/changeset/base/340552
Log:
MFC r339507:
amd64: flush L1 data cache on syscall return with an error.
Modified:
stable/12/sys/amd64/amd64/initcpu.c
stable/12/sys/amd64/amd64/machdep.c
stable/12/sys/amd64/amd64/support.S
stable/12/sys/amd64/amd64/trap.c
stable/12/sys/amd64/ia32/ia32_syscall.c
stable/12/sys/amd64/include/md_var.h
stable/12/sys/dev/cpuctl/cpuctl.c
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/sys/amd64/amd64/initcpu.c
==============================================================================
--- stable/12/sys/amd64/amd64/initcpu.c Sun Nov 18 09:48:51 2018
(r340551)
+++ stable/12/sys/amd64/amd64/initcpu.c Sun Nov 18 09:49:24 2018
(r340552)
@@ -253,6 +253,7 @@ initializecpu(void)
}
hw_ibrs_recalculate();
hw_ssb_recalculate(false);
+ amd64_syscall_ret_flush_l1d_recalc();
switch (cpu_vendor_id) {
case CPU_VENDOR_AMD:
init_amd();
Modified: stable/12/sys/amd64/amd64/machdep.c
==============================================================================
--- stable/12/sys/amd64/amd64/machdep.c Sun Nov 18 09:48:51 2018
(r340551)
+++ stable/12/sys/amd64/amd64/machdep.c Sun Nov 18 09:49:24 2018
(r340552)
@@ -1728,6 +1728,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
!= NULL)
vty_set_preferred(VTY_VT);
+ TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
+ TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
+ TUNABLE_INT_FETCH("machdep.syscall_ret_l1d_flush",
+ &syscall_ret_l1d_flush_mode);
+
finishidentcpu(); /* Final stage of CPU initialization */
initializecpu(); /* Initialize CPU registers */
@@ -1875,9 +1880,6 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
x86_init_fdt();
#endif
thread0.td_critnest = 0;
-
- TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
- TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
TSEXIT();
Modified: stable/12/sys/amd64/amd64/support.S
==============================================================================
--- stable/12/sys/amd64/amd64/support.S Sun Nov 18 09:48:51 2018
(r340551)
+++ stable/12/sys/amd64/amd64/support.S Sun Nov 18 09:49:24 2018
(r340552)
@@ -1556,3 +1556,10 @@ ENTRY(flush_l1d_sw)
ret
#undef L1D_FLUSH_SIZE
END(flush_l1d_sw)
+
+ENTRY(flush_l1d_sw_abi)
+ pushq %rbx
+ call flush_l1d_sw
+ popq %rbx
+ ret
+END(flush_l1d_sw_abi)
Modified: stable/12/sys/amd64/amd64/trap.c
==============================================================================
--- stable/12/sys/amd64/amd64/trap.c Sun Nov 18 09:48:51 2018
(r340551)
+++ stable/12/sys/amd64/amd64/trap.c Sun Nov 18 09:49:24 2018
(r340552)
@@ -1056,6 +1056,84 @@ cpu_fetch_syscall_args(struct thread *td)
#include "../../kern/subr_syscall.c"
+static void (*syscall_ret_l1d_flush)(void);
+int syscall_ret_l1d_flush_mode;
+
+static void
+flush_l1d_hw(void)
+{
+
+ wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
+}
+
+static void __inline
+amd64_syscall_ret_flush_l1d_inline(int error)
+{
+ void (*p)(void);
+
+ if (error != 0 && error != EEXIST && error != EAGAIN &&
+ error != EXDEV && error != ENOENT && error != ENOTCONN &&
+ error != EINPROGRESS) {
+ p = syscall_ret_l1d_flush;
+ if (p != NULL)
+ p();
+ }
+}
+
+void
+amd64_syscall_ret_flush_l1d(int error)
+{
+
+ amd64_syscall_ret_flush_l1d_inline(error);
+}
+
+void
+amd64_syscall_ret_flush_l1d_recalc(void)
+{
+ bool l1d_hw;
+
+ l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
+again:
+ switch (syscall_ret_l1d_flush_mode) {
+ case 0:
+ syscall_ret_l1d_flush = NULL;
+ break;
+ case 1:
+ syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
+ flush_l1d_sw_abi;
+ break;
+ case 2:
+ syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
+ break;
+ case 3:
+ syscall_ret_l1d_flush = flush_l1d_sw_abi;
+ break;
+ default:
+ syscall_ret_l1d_flush_mode = 1;
+ goto again;
+ }
+}
+
+static int
+machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
+{
+ int error, val;
+
+ val = syscall_ret_l1d_flush_mode;
+ error = sysctl_handle_int(oidp, &val, 0, req);
+ if (error != 0 || req->newptr == NULL)
+ return (error);
+ syscall_ret_l1d_flush_mode = val;
+ amd64_syscall_ret_flush_l1d_recalc();
+ return (0);
+}
+SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
+ CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
+ machdep_syscall_ret_flush_l1d, "I",
+ "Flush L1D on syscall return with error (0 - off, 1 - on, "
+ "2 - use hw only, 3 - use sw only");
+
+
/*
* System call handler for native binaries. The trap frame is already
* set up by the assembler trampoline and a pointer to it is saved in
@@ -1110,4 +1188,6 @@ amd64_syscall(struct thread *td, int traced)
*/
if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+
+ amd64_syscall_ret_flush_l1d_inline(error);
}
Modified: stable/12/sys/amd64/ia32/ia32_syscall.c
==============================================================================
--- stable/12/sys/amd64/ia32/ia32_syscall.c Sun Nov 18 09:48:51 2018
(r340551)
+++ stable/12/sys/amd64/ia32/ia32_syscall.c Sun Nov 18 09:49:24 2018
(r340552)
@@ -231,6 +231,7 @@ ia32_syscall(struct trapframe *frame)
}
syscallret(td, error);
+ amd64_syscall_ret_flush_l1d(error);
}
static void
Modified: stable/12/sys/amd64/include/md_var.h
==============================================================================
--- stable/12/sys/amd64/include/md_var.h Sun Nov 18 09:48:51 2018
(r340551)
+++ stable/12/sys/amd64/include/md_var.h Sun Nov 18 09:49:24 2018
(r340552)
@@ -41,6 +41,7 @@ extern int hw_lower_amd64_sharedpage;
extern int hw_ibrs_disable;
extern int hw_ssb_disable;
extern int nmi_flush_l1d_sw;
+extern int syscall_ret_l1d_flush_mode;
extern vm_paddr_t intel_graphics_stolen_base;
extern vm_paddr_t intel_graphics_stolen_size;
@@ -58,8 +59,11 @@ void amd64_conf_fast_syscall(void);
void amd64_db_resume_dbreg(void);
void amd64_lower_shared_page(struct sysentvec *);
void amd64_syscall(struct thread *td, int traced);
+void amd64_syscall_ret_flush_l1d(int error);
+void amd64_syscall_ret_flush_l1d_recalc(void);
void doreti_iret(void) __asm(__STRING(doreti_iret));
void doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault));
+void flush_l1d_sw_abi(void);
void ld_ds(void) __asm(__STRING(ld_ds));
void ld_es(void) __asm(__STRING(ld_es));
void ld_fs(void) __asm(__STRING(ld_fs));
Modified: stable/12/sys/dev/cpuctl/cpuctl.c
==============================================================================
--- stable/12/sys/dev/cpuctl/cpuctl.c Sun Nov 18 09:48:51 2018
(r340551)
+++ stable/12/sys/dev/cpuctl/cpuctl.c Sun Nov 18 09:49:24 2018
(r340552)
@@ -521,6 +521,9 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td
hw_ibrs_recalculate();
restore_cpu(oldcpu, is_bound, td);
hw_ssb_recalculate(true);
+#ifdef __amd64__
+ amd64_syscall_ret_flush_l1d_recalc();
+#endif
printcpuinfo();
return (0);
}
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
