Author: sir_richard
Date: Wed Jan 27 04:05:10 2010
New Revision: 45280
URL: http://svn.reactos.org/svn/reactos?rev=45280&view=rev
Log:
[NTOS]: Rework the trap macros again. This time we needed some added complexity
because of the need for runtime patching at boot. We got away with it nicely
and basically implement a simple "if" in ASM, which gives us a deterministic
set of instructions (vs the compiler). We then patch if if needed (see next
bullet).
[NTOS]: Support systems without SYSENTER. Nothing was actually disabling the
SYSEXIT mechanism recently enabled. Now IRET is always used unless a SYSENTER
machine is discovered, in which case the jmp to the IRET code is patched with a
jmp to SYSEXIT code.
[PERF]: Set WP0 bit in CR0 later, thus allowing the existing runtime patches
(and this new one) to function without requiring MmSetPageProtect. Saves TLB
flushes and page table walking/mapping on boot.
Left in some debug prints to see what buildbot reports...will remove if it
survives.
Modified:
trunk/reactos/ntoskrnl/include/internal/i386/ke.h
trunk/reactos/ntoskrnl/include/internal/trap_x.h
trunk/reactos/ntoskrnl/ke/i386/cpu.c
trunk/reactos/ntoskrnl/ke/i386/kiinit.c
Modified: trunk/reactos/ntoskrnl/include/internal/i386/ke.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/i386/ke.h?rev=45280&r1=45279&r2=45280&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/i386/ke.h [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/include/internal/i386/ke.h [iso-8859-1] Wed Jan 27
04:05:10 2010
@@ -456,6 +456,9 @@
extern VOID __cdecl CopyParams(VOID);
extern VOID __cdecl ReadBatch(VOID);
extern VOID __cdecl FrRestore(VOID);
+extern CHAR KiSystemCallExitBranch[];
+extern CHAR KiSystemCallExit[];
+extern CHAR KiSystemCallExit2[];
//
// Trap Macros
Modified: trunk/reactos/ntoskrnl/include/internal/trap_x.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/trap_x.h?rev=45280&r1=45279&r2=45280&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/trap_x.h [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/include/internal/trap_x.h [iso-8859-1] Wed Jan 27
04:05:10 2010
@@ -190,7 +190,7 @@
//
VOID
FORCEINLINE
-DECLSPEC_NORETURN
+/* Do not mark this as DECLSPEC_NORETURN because possibly executing code
follows it! */
KiSystemCallReturn(IN PKTRAP_FRAME TrapFrame)
{
/* Restore nonvolatiles, EAX, and do a "jump" back to the kernel caller */
@@ -205,6 +205,7 @@
"movl %c[e](%%esp), %%edx\n"
"addl $%c[v],%%esp\n" /* A WHOLE *KERNEL* frame since we're not
IRET'ing */
"jmp *%%edx\n"
+ ".globl _KiSystemCallExit2\n_KiSystemCallExit2:\n"
:
: "r"(TrapFrame),
[b] "i"(KTRAP_FRAME_EBX),
@@ -216,7 +217,6 @@
[v] "i"(KTRAP_FRAME_ESP)
: "%esp"
);
- UNREACHABLE;
}
VOID
@@ -227,6 +227,7 @@
/* Regular interrupt exit, but we only restore EAX as a volatile */
__asm__ __volatile__
(
+ ".globl _KiSystemCallExit\n_KiSystemCallExit:\n"
"movl %0, %%esp\n"
"movl %c[b](%%esp), %%ebx\n"
"movl %c[s](%%esp), %%esi\n"
@@ -405,6 +406,29 @@
asm volatile(".byte 0xC4\n.byte 0xC4\n");
}
+VOID
+FORCEINLINE
+KiUserSystemCall(IN PKTRAP_FRAME TrapFrame)
+{
+ /*
+ * Kernel call or user call?
+ *
+ * This decision is made in inlined assembly because we need to patch
+ * the relative offset of the user-mode jump to point to the SYSEXIT
+ * routine if the CPU supports it. The only way to guarantee that a
+ * relative jnz/jz instruction is generated is to force it with the
+ * inline assembler.
+ */
+ asm volatile
+ (
+ "test $1, %0\n" /* MODE_MASK */
+ ".globl _KiSystemCallExitBranch\n_KiSystemCallExitBranch:\n"
+ "jnz _KiSystemCallExit\n"
+ :
+ : "r"(TrapFrame->SegCs)
+ );
+}
+
//
// Generic Exit Routine
//
@@ -502,43 +526,35 @@
/* Check for system call -- a system call skips volatiles! */
if (__builtin_expect(SkipBits.SkipVolatiles, 0)) /* More INTs than
SYSCALLs */
{
- /* Kernel call or user call? */
- if (__builtin_expect(KiUserTrap(TrapFrame), 1)) /* More Ring 3 than 0
*/
+ /* User or kernel call? */
+ KiUserSystemCall(TrapFrame);
+
+ /* Restore EFLags */
+ __writeeflags(TrapFrame->EFlags);
+
+ /* Call is kernel, so do a jump back since this wasn't a real INT */
+ KiSystemCallReturn(TrapFrame);
+
+ /* If we got here, this is SYSEXIT: are we stepping code? */
+ if (!(TrapFrame->EFlags & EFLAGS_TF))
{
- /* Is SYSENTER supported and/or enabled, or are we stepping code?
*/
- if (__builtin_expect((KiFastSystemCallDisable) ||
- (TrapFrame->EFlags & EFLAGS_TF), 0))
- {
- /* Exit normally */
- KiSystemCallTrapReturn(TrapFrame);
- }
- else
- {
- /* Restore user FS */
- Ke386SetFs(KGDT_R3_TEB | RPL_MASK);
-
- /* Remove interrupt flag */
- TrapFrame->EFlags &= ~EFLAGS_INTERRUPT_MASK;
- __writeeflags(TrapFrame->EFlags);
-
- /* Exit through SYSEXIT */
- KiSystemCallSysExitReturn(TrapFrame);
- }
+ /* Restore user FS */
+ Ke386SetFs(KGDT_R3_TEB | RPL_MASK);
+
+ /* Remove interrupt flag */
+ TrapFrame->EFlags &= ~EFLAGS_INTERRUPT_MASK;
+ __writeeflags(TrapFrame->EFlags);
+
+ /* Exit through SYSEXIT */
+ KiSystemCallSysExitReturn(TrapFrame);
}
- else
- {
- /* Restore EFLags */
- __writeeflags(TrapFrame->EFlags);
-
- /* Call is kernel, so do a jump back since this wasn't a real INT
*/
- KiSystemCallReturn(TrapFrame);
- }
- }
- else
- {
- /* Return from interrupt */
- KiTrapReturn(TrapFrame);
- }
+
+ /* Exit through IRETD, either due to debugging or due to lack of
SYSEXIT */
+ KiSystemCallTrapReturn(TrapFrame);
+ }
+
+ /* Return from interrupt */
+ KiTrapReturn(TrapFrame);
}
//
Modified: trunk/reactos/ntoskrnl/ke/i386/cpu.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ke/i386/cpu.c?rev=45280&r1=45279&r2=45280&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/ke/i386/cpu.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/ke/i386/cpu.c [iso-8859-1] Wed Jan 27 04:05:10 2010
@@ -45,6 +45,15 @@
BOOLEAN KiI386PentiumLockErrataPresent;
BOOLEAN KiSMTProcessorsPresent;
+/* The distance between SYSEXIT and IRETD return modes */
+UCHAR KiSystemCallExitAdjust;
+
+/* The offset that was applied -- either 0 or the value above */
+UCHAR KiSystemCallExitAdjusted;
+
+/* Whether the adjustment was already done once */
+BOOLEAN KiFastCallCopyDoneOnce;
+
/* Flush data */
volatile LONG KiTbFlushTimeStamp;
@@ -801,16 +810,92 @@
VOID
NTAPI
+KiDisableFastSyscallReturn(VOID)
+{
+ /* Was it applied? */
+ if (KiSystemCallExitAdjusted)
+ {
+ /* Restore the original value */
+ KiSystemCallExitBranch[1] = KiSystemCallExitBranch[1] -
KiSystemCallExitAdjusted;
+
+ /* It's not adjusted anymore */
+ KiSystemCallExitAdjusted = FALSE;
+ }
+}
+
+VOID
+NTAPI
+KiEnableFastSyscallReturn(VOID)
+{
+ /* Check if the patch has already been done */
+ if ((KiSystemCallExitAdjusted == KiSystemCallExitAdjust) &&
+ (KiFastCallCopyDoneOnce))
+ {
+ DPRINT1("SYSEXIT Code Patch was already done!\n");
+ return;
+ }
+
+ /* Make sure the offset is within the distance of a Jxx SHORT */
+ if ((KiSystemCallExitBranch[1] - KiSystemCallExitAdjust) < 0x80)
+ {
+ /* Remove any existing code patch */
+ DPRINT1("Correct SHORT size found\n");
+ KiDisableFastSyscallReturn();
+
+ /* We should have a JNZ there */
+ ASSERT(KiSystemCallExitBranch[0] == 0x75);
+
+ /* Do the patch */
+ DPRINT1("Current jump offset: %lx\n", KiSystemCallExitBranch[1]);
+ KiSystemCallExitAdjusted = KiSystemCallExitAdjust;
+ KiSystemCallExitBranch[1] -= KiSystemCallExitAdjusted;
+ DPRINT1("New jump offset: %lx\n", KiSystemCallExitBranch[1]);
+
+ /* Remember that we've done it */
+ KiFastCallCopyDoneOnce = TRUE;
+ }
+ else
+ {
+ /* This shouldn't happen unless we've messed the macros up */
+ DPRINT1("Your compiled kernel is broken!\n");
+ DbgBreakPoint();
+ }
+}
+
+VOID
+NTAPI
KiRestoreFastSyscallReturnState(VOID)
{
- /* FIXME: NT has support for SYSCALL, IA64-SYSENTER, etc. */
-
/* Check if the CPU Supports fast system call */
if (KeFeatureBits & KF_FAST_SYSCALL)
{
+ /* Check if it has been disabled */
+ if (!KiFastSystemCallDisable)
+ {
+ /* KiSystemCallExit2 should come BEFORE KiSystemCallExit */
+ DPRINT1("Exit2: %p Exit1: %p\n", KiSystemCallExit2,
KiSystemCallExit);
+ ASSERT(KiSystemCallExit2 < KiSystemCallExit);
+
+ /* It's enabled, so we'll have to do a code patch */
+ KiSystemCallExitAdjust = KiSystemCallExit - KiSystemCallExit2;
+ DPRINT1("SYSENTER Capable Machine. Jump Offset Delta: %lx\n",
KiSystemCallExitAdjust);
+ }
+ else
+ {
+ /* Disable fast system call */
+ KeFeatureBits &= ~KF_FAST_SYSCALL;
+ }
+ }
+
+ /* Now check if all CPUs support fast system call, and the registry allows
it */
+ if (KeFeatureBits & KF_FAST_SYSCALL)
+ {
/* Do an IPI to enable it */
KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
}
+
+ /* Perform the code patch that is required */
+ KiEnableFastSyscallReturn();
}
ULONG_PTR
Modified: trunk/reactos/ntoskrnl/ke/i386/kiinit.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ke/i386/kiinit.c?rev=45280&r1=45279&r2=45280&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/ke/i386/kiinit.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/ke/i386/kiinit.c [iso-8859-1] Wed Jan 27 04:05:10
2010
@@ -31,7 +31,6 @@
NTAPI
KiInitMachineDependent(VOID)
{
- ULONG Protect;
ULONG CpuCount;
BOOLEAN FbCaching = FALSE;
NTSTATUS Status;
@@ -153,12 +152,7 @@
/* FIXME: Implement and enable XMM Page Zeroing for Mm */
/* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
- Protect = MmGetPageProtect(NULL, RtlPrefetchMemoryNonTemporal);
- MmSetPageProtect(NULL,
- RtlPrefetchMemoryNonTemporal,
- Protect | PAGE_IS_WRITABLE);
*(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
- MmSetPageProtect(NULL, RtlPrefetchMemoryNonTemporal, Protect);
}
}
@@ -320,6 +314,9 @@
/* FIXME: TODO */
DPRINT1("ISR Time Limit not yet supported\n");
}
+
+ /* Set CR0 features based on detected CPU */
+ KiSetCR0Bits();
}
VOID
@@ -400,9 +397,6 @@
/* Detect and set the CPU Type */
KiSetProcessorType();
-
- /* Set CR0 features based on detected CPU */
- KiSetCR0Bits();
/* Check if an FPU is present */
NpxPresent = KiIsNpxPresent();
