Module Name: src
Committed By: scole
Date: Sat Apr 8 17:42:47 UTC 2017
Modified Files:
src/sys/arch/ia64/ia64: exception.S
Log Message:
Updates from FreeBSD, mostly compile-tested
To generate a diff of this commit:
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/ia64/ia64/exception.S
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/ia64/ia64/exception.S
diff -u src/sys/arch/ia64/ia64/exception.S:1.5 src/sys/arch/ia64/ia64/exception.S:1.6
--- src/sys/arch/ia64/ia64/exception.S:1.5 Sat Oct 1 15:59:27 2011
+++ src/sys/arch/ia64/ia64/exception.S Sat Apr 8 17:42:47 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: exception.S,v 1.5 2011/10/01 15:59:27 chs Exp $ */
+/* $NetBSD: exception.S,v 1.6 2017/04/08 17:42:47 scole Exp $ */
/*-
* Copyright (c) 2003,2004 Marcel Moolenaar
@@ -28,18 +28,206 @@
*/
#include <machine/asm.h>
-#include <machine/pte.h>
+/* __FBSDID("$FreeBSD: releng/10.1/sys/ia64/ia64/exception.S 268200 2014-07-02 23:47:43Z marcel $"); */
#include "assym.h"
/*
+ * Nested TLB restart tokens. These are used by the
+ * nested TLB handler for jumping back to the code
+ * where the nested TLB was caused.
+ */
+#define NTLBRT_SAVE 0x12c12c
+#define NTLBRT_RESTORE 0x12c12d
+
+/*
* ar.k7 = kernel memory stack
* ar.k6 = kernel register stack
* ar.k5 = EPC gateway page
* ar.k4 = PCPU data
*/
- .text
+ .section .ivt.data, "aw"
+
+ .align 8
+ .global ia64_kptdir
+ .size ia64_kptdir, 8
+ia64_kptdir: data8 0
+
+
+#ifdef XTRACE
+
+ .align 8
+ .global ia64_xtrace_mask
+ .size ia64_xtrace_mask, 8
+ia64_xtrace_mask: data8 0
+
+ .align 4
+ .global ia64_xtrace_enabled
+ .size ia64_xtrace_enabled, 4
+ia64_xtrace_enabled: data4 0
+
+#define XTRACE_HOOK(offset) \
+{ .mii ; \
+ nop 0 ; \
+ mov r31 = b7 ; \
+ mov r28 = pr ; \
+} ; \
+{ .mib ; \
+ nop 0 ; \
+ mov r25 = ip ; \
+ br.sptk ia64_xtrace_write ;; \
+} ; \
+{ .mii ; \
+ nop 0 ; \
+ mov b7 = r31 ; \
+ mov pr = r28, 0x1ffff ;; \
+}
+
+ .section .ivt.text, "ax"
+
+// We can only use r25, r26 & r27
+ENTRY_NOPROFILE(ia64_xtrace_write, 0)
+{ .mlx
+ add r25 = 16, r25
+ movl r26 = ia64_xtrace_enabled
+ ;;
+}
+{ .mmi
+ mov r27 = ar.k3
+ ld4 r26 = [r26]
+ mov b7 = r25
+ ;;
+}
+{ .mib
+ add r25 = -32, r25
+ cmp.eq p15,p0 = r0, r26
+(p15) br.dptk.few b7
+ ;;
+}
+{ .mib
+ nop 0
+ cmp.eq p15,p0 = r0, r27
+(p15) br.dptk.few b7
+ ;;
+}
+{ .mmi
+ st8 [r27] = r25, 8 // 0x00 IVT
+ mov r26 = ar.itc
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r26, 8 // 0x08 ITC
+ mov r25 = cr.iip
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r25, 8 // 0x10 IIP
+ mov r26 = cr.ifa
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r26, 8 // 0x18 IFA
+ mov r25 = cr.isr
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r25, 8 // 0x20 ISR
+ mov r26 = cr.ipsr
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r26, 8 // 0x28 IPSR
+ mov r25 = cr.itir
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r25, 8 // 0x30 ITIR
+ mov r26 = cr.iipa
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r26, 8 // 0x38 IIPA
+ mov r25 = cr.ifs
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r25, 8 // 0x40 IFS
+ mov r26 = cr.iim
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r26, 8 // 0x48 IIM
+ mov r25 = cr.iha
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r25, 8 // 0x50 IHA
+ mov r26 = ar.unat
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r26, 8 // 0x58 UNAT
+ mov r25 = ar.rsc
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r25, 8 // 0x60 RSC
+ mov r26 = ar.bsp
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r26, 8 // 0x68 BSP
+ mov r25 = r13
+ nop 0
+ ;;
+}
+{ .mmi
+ st8 [r27] = r25, 8 // 0x70 PCPU/TLS
+ mov r26 = r12
+ nop 0
+ ;;
+}
+{ .mlx
+ st8 [r27] = r26, 8 // 0x78 SP
+ movl r25 = ia64_xtrace_mask
+ ;;
+}
+{ .mmi
+ ld8 r26 = [r25]
+ ;;
+ and r25 = r27, r26
+ nop 0
+ ;;
+}
+{ .mib
+ mov ar.k3 = r25
+ nop 0
+ br.sptk b7
+ ;;
+}
+END(ia64_xtrace_write)
+
+#else /* XTRACE */
+
+#define XTRACE_HOOK(offset)
+
+ .section .ivt.text, "ax"
+
+#endif /* XTRACE */
/*
* exception_save: save interrupted state
@@ -67,7 +255,7 @@ ENTRY_NOPROFILE(exception_save, 0)
;;
}
{ .mmi
- cmp.le p14,p15=5,r31
+ cmp.le p14,p15=IA64_VM_MINKERN_REGION,r31
;;
(p15) mov r23=ar.k7 // kernel memory stack
(p14) mov r23=sp
@@ -82,42 +270,57 @@ ENTRY_NOPROFILE(exception_save, 0)
}
{ .mmi
mov ar.rsc=0
- sub r19=r23,r30
- add r31=8,r30
- ;;
-}
-{ .mlx
mov r22=cr.iip
- movl r26=exception_save_restart
+ addl r29=NTLBRT_SAVE,r0 // 22-bit restart token.
;;
}
/*
- * We have a 1KB aligned trapframe, pointed to by sp. If we write
- * to the trapframe, we may trigger a data nested TLB fault. By
- * aligning the trapframe on a 1KB boundary, we guarantee that if
- * we get a data nested TLB fault, it will be on the very first
- * write. Since the data nested TLB fault does not preserve any
- * state, we have to be careful what we clobber. Consequently, we
- * have to be careful what we use here. Below a list of registers
- * that are currently alive:
+ * We have a 1KB aligned trapframe, pointed to by r30. We can't
+ * reliably write to the trapframe using virtual addressing, due
+ * to the fact that TC entries we depend on can be removed by:
+ * 1. ptc.g instructions issued by other threads/cores/CPUs, or
+ * 2. TC modifications in another thread on the same core.
+ * When our TC entry gets removed, we get nested TLB faults and
+ * since no state is saved, we can only deal with those when
+ * explicitly coded and expected.
+ * As such, we switch to physical addressing and account for the
+ * fact that the tpa instruction can cause a nested TLB fault.
+ * Since the data nested TLB fault does not preserve any state,
+ * we have to be careful what we clobber. Consequently, we have
+ * to be careful what we use here. Below a list of registers that
+ * are considered alive:
* r16,r17=arguments
* r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS
- * r26=restart point
- * r30,r31=trapframe pointers
+ * r29=restart token
+ * r30=trapframe pointers
* p14,p15=memory stack switch
*/
exception_save_restart:
+ tpa r24=r30 // Nested TLB fault possible
+ sub r19=r23,r30
+ nop 0
+ ;;
+
+ rsm psr.dt
+ add r29=16,r19 // Clobber restart token
+ mov r30=r24
+ ;;
+ srlz.d
+ add r31=8,r24
+ ;;
+
+ // r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS
+ // r29=delta
{ .mmi
st8 [r30]=r19,16 // length
st8 [r31]=r0,16 // flags
- add r19=16,r19
;;
}
{ .mmi
st8.spill [r30]=sp,16 // sp
st8 [r31]=r20,16 // unat
- sub sp=r23,r19
+ sub sp=r23,r29
;;
}
{ .mmi
@@ -127,6 +330,7 @@ exception_save_restart:
;;
}
// r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp
+ // r24=pfs
{ .mmi
st8 [r30]=r23,16 // rp
st8 [r31]=r18,16 // pr
@@ -149,7 +353,7 @@ exception_save_restart:
{ .mmi
st8 [r30]=r19,16 // rnat
st8 [r31]=r0,16 // __spare
- cmp.le p12,p13=5,r24
+ cmp.le p12,p13=IA64_VM_MINKERN_REGION,r24
;;
}
{ .mmi
@@ -165,7 +369,7 @@ exception_save_restart:
(p13) dep r20=r20,r21,0,9 // align dirty registers
;;
}
- // r20=bspstore, r22=iip, r23=ipsr
+ // r19=rnat, r20=bspstore, r22=iip, r23=ipsr
{ .mmi
st8 [r31]=r23,16 // psr
(p13) mov ar.bspstore=r20
@@ -173,35 +377,34 @@ exception_save_restart:
;;
}
{ .mmi
+(p13) mov ar.rnat=r19
mov r18=ar.bsp
- ;;
- mov r19=cr.ifs
- sub r18=r18,r20
+ nop 0
;;
}
{ .mmi
+ mov r19=cr.ifs
st8.spill [r30]=gp,16 // gp
- st8 [r31]=r18,16 // ndirty
- nop 0
+ sub r18=r18,r20
;;
}
- // r19=ifs, r22=iip
+ // r18=ndirty, r19=ifs, r22=iip
{ .mmi
+ st8 [r31]=r18,16 // ndirty
st8 [r30]=r19,16 // cfm
- st8 [r31]=r22,16 // iip
nop 0
;;
}
{ .mmi
- st8 [r30]=r17 // ifa
mov r18=cr.isr
+ st8 [r31]=r22,16 // iip
add r29=16,r30
;;
}
{ .mmi
- st8 [r31]=r18 // isr
- add r30=8,r29
- add r31=16,r29
+ st8 [r30]=r17,24 // ifa
+ st8 [r31]=r18,24 // isr
+ nop 0
;;
}
{ .mmi
@@ -341,11 +544,11 @@ exception_save_restart:
;;
}
{ .mlx
- ssm psr.ic|psr.dfh
+ ssm psr.dt|psr.ic|psr.dfh
movl gp=__gp
;;
}
-{ .mfb
+{ .mib
srlz.d
nop 0
br.sptk b7
@@ -366,29 +569,48 @@ ENTRY_NOPROFILE(exception_restore, 0)
nop 0
;;
}
-{ .mmi
- add r3=SIZEOF_TRAPFRAME-32,sp
- add r2=SIZEOF_TRAPFRAME-16,sp
- add r8=SIZEOF_SPECIAL+16,sp
+
+ // The next instruction can fault. Let it be...
+ tpa r9=sp
;;
-}
- // The next load can trap. Let it be...
+ rsm psr.dt|psr.ic
+ add r8=SIZEOF_SPECIAL+16,r9
+ ;;
+ srlz.d
+ add r2=SIZEOF_TRAPFRAME-16,r9
+ add r3=SIZEOF_TRAPFRAME-32,r9
+ ;;
+
+{ .mmi
ldf.fill f15=[r2],-32 // f15
ldf.fill f14=[r3],-32 // f14
+ nop 0
;;
+}
+{ .mmi
ldf.fill f13=[r2],-32 // f13
ldf.fill f12=[r3],-32 // f12
+ nop 0
;;
+}
+{ .mmi
ldf.fill f11=[r2],-32 // f11
ldf.fill f10=[r3],-32 // f10
+ nop 0
;;
+}
+{ .mmi
ldf.fill f9=[r2],-32 // f9
ldf.fill f8=[r3],-32 // f8
+ nop 0
;;
+}
+{ .mmi
ldf.fill f7=[r2],-24 // f7
ldf.fill f6=[r3],-16 // f6
+ nop 0
;;
-
+}
{ .mmi
ld8 r8=[r8] // unat (after)
;;
@@ -445,53 +667,53 @@ ENTRY_NOPROFILE(exception_restore, 0)
bsw.0
;;
}
+{ .mii
+ ld8 r16=[r9] // tf_length
+ add r31=16,r9
+ add r30=24,r9
+}
{ .mmi
ld8.fill r15=[r3],-16 // r15
ld8.fill r14=[r2],-16 // r14
- add r31=16,sp
+ nop 0
;;
}
{ .mmi
- ld8 r16=[sp] // tf_length
ld8.fill r11=[r3],-16 // r11
- add r30=24,sp
- ;;
-}
-{ .mmi
ld8.fill r10=[r2],-16 // r10
- ld8.fill r9=[r3],-16 // r9
add r16=r16,sp // ar.k7
;;
}
{ .mmi
+ ld8.fill r9=[r3],-16 // r9
ld8.fill r8=[r2],-16 // r8
- ld8.fill r3=[r3] // r3
+ nop 0
;;
}
- // We want nested TLB faults from here on...
- rsm psr.ic|psr.i
+{ .mmi
+ ld8.fill r3=[r3] // r3
ld8.fill r2=[r2] // r2
nop 0
;;
- srlz.d
- ld8.fill sp=[r31],16 // sp
- nop 0
- ;;
+}
+ ld8.fill sp=[r31],16 // sp
ld8 r17=[r30],16 // unat
- ld8 r29=[r31],16 // rp
;;
+ ld8 r29=[r31],16 // rp
ld8 r18=[r30],16 // pr
+ ;;
ld8 r28=[r31],16 // pfs
+ ld8 r20=[r30],24 // bspstore
mov rp=r29
;;
- ld8 r20=[r30],24 // bspstore
ld8 r21=[r31],24 // rnat
mov ar.pfs=r28
;;
- ld8.fill r29=[r30],16 // tp
+ ld8.fill r26=[r30],16 // tp
ld8 r22=[r31],16 // rsc
;;
+
{ .mmi
ld8 r23=[r30],16 // fpsr
ld8 r24=[r31],16 // psr
@@ -500,21 +722,21 @@ ENTRY_NOPROFILE(exception_restore, 0)
}
{ .mmi
ld8.fill r1=[r30],16 // gp
- ld8 r25=[r31],16 // ndirty
- cmp.le p14,p15=5,r28
+ ld8 r27=[r31],16 // ndirty
+ cmp.le p14,p15=IA64_VM_MINKERN_REGION,r28
;;
}
-{ .mmb
- ld8 r26=[r30] // cfm
+{ .mmi
+ ld8 r25=[r30] // cfm
ld8 r19=[r31] // ip
nop 0
;;
}
-{ .mib
+{ .mii
// Switch register stack
alloc r30=ar.pfs,0,0,0,0 // discard current frame
- shl r31=r25,16 // value for ar.rsc
- nop 0
+ shl r31=r27,16 // value for ar.rsc
+(p15) mov r13=r26
;;
}
// The loadrs can fault if the backing store is not currently
@@ -525,15 +747,21 @@ ENTRY_NOPROFILE(exception_restore, 0)
{ .mmi
mov ar.rsc=r31 // setup for loadrs
mov ar.k7=r16
-(p15) mov r13=r29
+ addl r29=NTLBRT_RESTORE,r0 // 22-bit restart token
;;
}
+
+ ssm psr.dt
+ ;;
+ srlz.d
+ mov r16 = r25
+
exception_restore_restart:
{ .mmi
mov r30=ar.bspstore
;;
loadrs // load user regs
- nop 0
+ mov r29=0 // Clobber restart token
;;
}
{ .mmi
@@ -543,21 +771,22 @@ exception_restore_restart:
dep r31=0,r31,0,13 // 8KB aligned
;;
}
-{ .mmb
+{ .mmi
+ mov cr.ifs=r16
mov ar.k6=r31
- mov ar.rnat=r21
- nop 0
+ mov pr=r18,0x1ffff
;;
}
-{ .mmb
- mov ar.unat=r17
+{ .mmi
mov cr.iip=r19
+ mov ar.unat=r17
nop 0
+ ;;
}
{ .mmi
mov cr.ipsr=r24
- mov cr.ifs=r26
- mov pr=r18,0x1fffe
+ mov ar.rnat=r21
+ nop 0
;;
}
{ .mmb
@@ -581,22 +810,22 @@ END(exception_restore)
{ .mib ; \
mov r17=_ifa_ ; \
mov r16=ip ; \
- br.sptk exception_save ; \
+ br.sptk exception_save ;; \
} ; \
{ .mmi ; \
-(p11) ssm psr.i ;; \
- alloc r15=ar.pfs,0,0,2,0 ; \
+ alloc r15=ar.pfs,0,0,2,0 ;; \
+(p11) ssm psr.i ; \
mov out0=_n_ ;; \
} ; \
-{ .mfb ; \
+{ .mib ; \
+(p11) srlz.d ; \
add out1=16,sp ; \
- nop 0 ; \
- br.call.sptk rp=_func_ ; \
+ br.call.sptk rp=_func_ ;; \
} ; \
-{ .mfb ; \
+{ .mib ; \
nop 0 ; \
nop 0 ; \
- br.sptk exception_restore ; \
+ br.sptk exception_restore ;; \
}
#define IVT_ENTRY(name, offset) \
@@ -607,7 +836,8 @@ END(exception_restore)
.unwabi @svr4, 'I'; \
.save rp, r0; \
.body; \
-ivt_##name:
+ivt_##name: \
+ XTRACE_HOOK(offset)
#define IVT_END(name) \
.endp ivt_##name
@@ -623,7 +853,7 @@ ivt_##name:
* bundles per vector and 48 slots with 16 bundles per vector.
*/
- .section .text.ivt,"ax"
+ .section .ivt, "ax"
.align 32768
.global ia64_vector_table
@@ -645,6 +875,7 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
add r20=24,r18 // collision chain
;;
ld8 r21=[r21] // check VHPT tag
+ ld8 r20=[r20] // bucket head
;;
cmp.ne p15,p0=r21,r19
(p15) br.dpnt.few 1f
@@ -652,15 +883,15 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
ld8 r21=[r18] // read pte
;;
itc.i r21 // insert pte
- ;;
mov pr=r17,0x1ffff
+ ;;
rfi // done
;;
-1: ld8 r20=[r20] // first entry
- ;;
- rsm psr.dt // turn off data translations
+1: rsm psr.dt // turn off data translations
+ dep r20=0,r20,61,3 // convert vhpt ptr to physical
;;
srlz.d // serialize
+ ld8 r20=[r20] // first entry
;;
2: cmp.eq p15,p0=r0,r20 // done?
(p15) br.cond.spnt.few 9f // bail if done
@@ -672,7 +903,12 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
cmp.ne p15,p0=r21,r19 // compare tags
(p15) br.cond.sptk.few 3f // if not, read next in chain
;;
- ld8 r21=[r20],8 // read pte
+ ld8 r21=[r20] // read pte
+ mov r22=PTE_ACCESSED
+ ;;
+ or r21=r21,r22
+ ;;
+ st8 [r20]=r21,8
;;
ld8 r22=[r20] // read rest of pte
;;
@@ -695,17 +931,19 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
st8.rel [r18]=r19 // store new tag
;;
itc.i r21 // and place in TLB
+ ssm psr.dt
;;
+ srlz.d
mov pr=r17,0x1ffff // restore predicates
rfi
-
+ ;;
3: add r20=24,r20 // next in chain
;;
ld8 r20=[r20] // read chain
- br.cond.sptk.few 2b // loop
-
-9: mov pr=r17,0x1ffff // restore predicates
- ssm psr.dt
+ br.sptk 2b // loop
+ ;;
+9: ssm psr.dt
+ mov pr=r17,0x1ffff // restore predicates
;;
srlz.d
;;
@@ -731,15 +969,15 @@ IVT_ENTRY(Data_TLB, 0x0800)
ld8 r21=[r18] // read pte
;;
itc.d r21 // insert pte
- ;;
mov pr=r17,0x1ffff
+ ;;
rfi // done
;;
-1: ld8 r20=[r20] // first entry
+1: rsm psr.dt // turn off data translations
+ dep r20=0,r20,61,3 // convert vhpt ptr to physical
;;
- rsm psr.dt // turn off data translations
- ;;
srlz.d // serialize
+ ld8 r20=[r20] // first entry
;;
2: cmp.eq p15,p0=r0,r20 // done?
(p15) br.cond.spnt.few 9f // bail if done
@@ -751,7 +989,12 @@ IVT_ENTRY(Data_TLB, 0x0800)
cmp.ne p15,p0=r21,r19 // compare tags
(p15) br.cond.sptk.few 3f // if not, read next in chain
;;
- ld8 r21=[r20],8 // read pte
+ ld8 r21=[r20] // read pte
+ mov r22=PTE_ACCESSED
+ ;;
+ or r21=r21,r22
+ ;;
+ st8 [r20]=r21,8
;;
ld8 r22=[r20] // read rest of pte
;;
@@ -774,17 +1017,19 @@ IVT_ENTRY(Data_TLB, 0x0800)
st8.rel [r18]=r19 // store new tag
;;
itc.d r21 // and place in TLB
+ ssm psr.dt
;;
+ srlz.d
mov pr=r17,0x1ffff // restore predicates
rfi
-
+ ;;
3: add r20=24,r20 // next in chain
;;
ld8 r20=[r20] // read chain
- br.cond.sptk.few 2b // loop
-
-9: mov pr=r17,0x1ffff // restore predicates
- ssm psr.dt
+ br.sptk 2b // loop
+ ;;
+9: ssm psr.dt
+ mov pr=r17,0x1ffff // restore predicates
;;
srlz.d
;;
@@ -796,24 +1041,40 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0
mov r18=pr // save predicates
;;
extr.u r17=r16,61,3 // get region number
+ mov r19=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX
;;
- cmp.ge p13,p0=5,r17 // RR0-RR5?
- cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14
-(p13) br.spnt 9f
+ cmp.eq p13,p0=IA64_PBVM_RR,r17 // RR4?
+(p13) br.cond.sptk.few 4f
;;
-(p15) movl r17=PTE_PRESENT+PTE_MA_WB+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+ \
- PTE_AR_RX
-(p14) movl r17=PTE_PRESENT+PTE_MA_UC+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+ \
- PTE_AR_RX
+ cmp.ge p13,p0=5,r17 // RR0-RR5?
+ cmp.eq p14,p15=7,r17 // RR7?
+(p13) br.cond.spnt.few 9f
;;
- dep r16=0,r16,50,14 // clear bits above PPN
+(p14) add r19=PTE_MA_WB,r19
+(p15) add r19=PTE_MA_UC,r19
+ dep r17=0,r16,50,14 // clear bits above PPN
;;
- dep r16=r17,r16,0,12 // put pte bits in 0..11
+1: dep r16=r19,r17,0,12 // put pte bits in 0..11
;;
itc.i r16
mov pr=r18,0x1ffff // restore predicates
;;
rfi
+ ;;
+4:
+ add r19=PTE_MA_WB,r19
+ movl r17=IA64_PBVM_BASE
+ ;;
+ sub r17=r16,r17
+ movl r16=IA64_PBVM_PGTBL
+ ;;
+ extr.u r17=r17,IA64_PBVM_PAGE_SHIFT,61-IA64_PBVM_PAGE_SHIFT
+ ;;
+ shladd r16=r17,3,r16
+ ;;
+ ld8 r17=[r16]
+ br.sptk 1b
+ ;;
9: mov pr=r18,0x1ffff // restore predicates
CALL(trap, 3, cr.ifa)
IVT_END(Alternate_Instruction_TLB)
@@ -823,24 +1084,40 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000)
mov r18=pr // save predicates
;;
extr.u r17=r16,61,3 // get region number
+ mov r19=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX
;;
- cmp.ge p13,p0=5,r17 // RR0-RR5?
- cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14
-(p13) br.spnt 9f
+ cmp.eq p13,p0=IA64_PBVM_RR,r17 // RR4?
+(p13) br.cond.sptk.few 4f
;;
-(p15) movl r17=PTE_PRESENT+PTE_MA_WB+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+ \
- PTE_AR_RW
-(p14) movl r17=PTE_PRESENT+PTE_MA_UC+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+ \
- PTE_AR_RW
+ cmp.ge p13,p0=5,r17 // RR0-RR5?
+ cmp.eq p14,p15=7,r17 // RR7?
+(p13) br.cond.spnt.few 9f
;;
- dep r16=0,r16,50,14 // clear bits above PPN
+(p14) add r19=PTE_MA_WB,r19
+(p15) add r19=PTE_MA_UC,r19
+ dep r17=0,r16,50,14 // clear bits above PPN
;;
- dep r16=r17,r16,0,12 // put pte bits in 0..11
+1: dep r16=r19,r17,0,12 // put pte bits in 0..11
;;
itc.d r16
mov pr=r18,0x1ffff // restore predicates
;;
rfi
+ ;;
+4:
+ add r19=PTE_MA_WB,r19
+ movl r17=IA64_PBVM_BASE
+ ;;
+ sub r17=r16,r17
+ movl r16=IA64_PBVM_PGTBL
+ ;;
+ extr.u r17=r17,IA64_PBVM_PAGE_SHIFT,61-IA64_PBVM_PAGE_SHIFT
+ ;;
+ shladd r16=r17,3,r16
+ ;;
+ ld8 r17=[r16]
+ br.sptk 1b
+ ;;
9: mov pr=r18,0x1ffff // restore predicates
CALL(trap, 4, cr.ifa)
IVT_END(Alternate_Data_TLB)
@@ -858,68 +1135,115 @@ IVT_ENTRY(Data_Nested_TLB, 0x1400)
// double nested faults. Since all virtual addresses we encounter
// here are direct mapped region 7 addresses, we have no problem
// constructing physical addresses.
-{ .mlx
- rsm psr.dt
- movl r27=ia64_kptdir
+
+{ .mmi
+ mov cr.ifa=r30
+ mov r26=rr[r30]
+ extr.u r27=r30,61,3
;;
}
{ .mii
- srlz.d
- dep r27=0,r27,61,3
- extr.u r28=r30,PAGE_SHIFT,61-PAGE_SHIFT
+ nop 0
+ dep r26=0,r26,0,2
+ cmp.eq p12,p13=7,r27
;;
}
{ .mii
- ld8 r27=[r27]
- shr.u r29=r28,PAGE_SHIFT-5 // dir index
- extr.u r28=r28,0,PAGE_SHIFT-5 // pte index
+ mov cr.itir=r26
+(p12) dep r28=0,r30,0,12
+(p13) extr.u r28=r30,3*PAGE_SHIFT-8, PAGE_SHIFT-3 // dir L0 index
+ ;;
+}
+{ .mlx
+(p12) add r28=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX+PTE_MA_WB,r28
+(p13) movl r27=ia64_kptdir
+ ;;
+}
+{ .mib
+(p13) ld8 r27=[r27]
+(p13) extr.u r26=r30,2*PAGE_SHIFT-5, PAGE_SHIFT-3 // dir L1 index
+(p12) br.cond.spnt.few 1f
;;
}
{ .mmi
- shladd r27=r29,3,r27
+ rsm psr.dt
;;
- mov cr.ifa=r30
+ srlz.d
dep r27=0,r27,61,3
;;
}
{ .mmi
- ld8 r27=[r27]
- mov r29=rr[r30]
- shl r28=r28,5
+ shladd r27=r28,3,r27
+ ;;
+ ld8 r27=[r27] // dir L1 page
+ extr.u r28=r30,PAGE_SHIFT,PAGE_SHIFT-5 // pte index
;;
}
{ .mii
- add r27=r27,r28 // address of pte
- dep r29=0,r29,0,2
+ shladd r27=r26,3,r27
+ shl r28=r28,5
;;
dep r27=0,r27,61,3
;;
}
-{ .mmi
- ld8 r28=[r27]
+ ld8 r27=[r27] // pte page
+ ;;
+ add r27=r28,r27
+ ;;
+ dep r27=0,r27,61,3
+ ;;
+ ld8 r28=[r27] // pte
;;
- mov cr.itir=r29
or r28=PTE_DIRTY+PTE_ACCESSED,r28
;;
-}
-{ .mlx
st8 [r27]=r28
- movl r29=exception_save_restart
;;
-}
+ ssm psr.dt
+ ;;
+1:
{ .mmi
itc.d r28
;;
- ssm psr.dt
- cmp.eq p12,p13=r26,r29
+ addl r26=NTLBRT_SAVE,r0
+ addl r27=NTLBRT_RESTORE,r0
;;
}
-{ .mbb
+{ .mmi
srlz.d
-(p12) br.sptk exception_save_restart
-(p13) br.sptk exception_restore_restart
+ cmp.eq p12,p0=r29,r26
+ cmp.eq p13,p0=r29,r27
+ ;;
+}
+{ .mbb
+ nop 0
+(p12) br.cond.sptk.few exception_save_restart
+(p13) br.cond.sptk.few exception_restore_restart
+ ;;
+}
+
+{ .mlx
+ mov r26=ar.bsp
+ movl r29=kstack
+ ;;
+}
+{ .mlx
+ mov r28=sp
+ movl r27=kstack_top
+ ;;
+}
+{ .mmi
+ add sp=-16,r27
+ ;;
+ mov r27=ar.bspstore
+ nop 0
;;
}
+ mov ar.rsc=0
+ dep r29=r27,r29,0,9
+ ;;
+ mov ar.bspstore=r29
+ ;;
+ CALL(trap, 5, r30)
IVT_END(Data_Nested_TLB)
IVT_ENTRY(Instruction_Key_Miss, 0x1800)
@@ -986,7 +1310,7 @@ IVT_ENTRY(Dirty_Bit, 0x2000)
;;
itc.d r21 // and place in TLB
ssm psr.dt
- ;;
+ ;;
srlz.d
mov pr=r17,0x1ffff // restore predicates
rfi
@@ -994,7 +1318,7 @@ IVT_ENTRY(Dirty_Bit, 0x2000)
2: add r20=24,r20 // next in chain
;;
ld8 r20=[r20] // read chain
- br.cond.sptk.few 1b // loop
+ br.sptk 1b // loop
;;
9: ssm psr.dt
mov pr=r17,0x1ffff // restore predicates
@@ -1013,11 +1337,13 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400
ttag r19=r16
add r20=24,r18 // collision chain
;;
- ld8 r20=[r20] // first entry
+ ld8 r20=[r20] // bucket head
;;
rsm psr.dt // turn off data translations
+ dep r20=0,r20,61,3 // convert vhpt ptr to physical
;;
srlz.d // serialize
+ ld8 r20=[r20] // first entry
;;
1: cmp.eq p15,p0=r0,r20 // done?
(p15) br.cond.spnt.few 9f // bail if done
@@ -1057,16 +1383,22 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400
st8.rel [r18]=r19 // store new tag
;;
itc.i r21 // and place in TLB
+ ssm psr.dt
;;
+ srlz.d
mov pr=r17,0x1ffff // restore predicates
rfi // walker will retry the access
-
+ ;;
2: add r20=24,r20 // next in chain
;;
ld8 r20=[r20] // read chain
- br.cond.sptk.few 1b // loop
-
-9: mov pr=r17,0x1ffff // restore predicates
+ br.sptk 1b // loop
+ ;;
+9: ssm psr.dt
+ mov pr=r17,0x1ffff // restore predicates
+ ;;
+ srlz.d
+ ;;
CALL(trap, 9, cr.ifa)
IVT_END(Instruction_Access_Bit)
@@ -1075,8 +1407,8 @@ IVT_ENTRY(Data_Access_Bit, 0x2800)
mov r17=pr
;;
thash r18=r16
- ttag r19=r16
;;
+ ttag r19=r16
add r20=24,r18 // collision chain
;;
ld8 r20=[r20] // bucket head
@@ -1127,16 +1459,20 @@ IVT_ENTRY(Data_Access_Bit, 0x2800)
itc.d r21 // and place in TLB
ssm psr.dt
;;
+ srlz.d
mov pr=r17,0x1ffff // restore predicates
rfi // walker will retry the access
-
+ ;;
2: add r20=24,r20 // next in chain
;;
ld8 r20=[r20] // read chain
- br.cond.sptk.few 1b // loop
-
+ br.sptk 1b // loop
+ ;;
9: ssm psr.dt
mov pr=r17,0x1ffff // restore predicates
+ ;;
+ srlz.d
+ ;;
CALL(trap, 10, cr.ifa)
IVT_END(Data_Access_Bit)
@@ -1150,17 +1486,23 @@ IVT_ENTRY(Break_Instruction, 0x2c00)
{ .mmi
alloc r15=ar.pfs,0,0,2,0
;;
- flushrs
+(p11) ssm psr.i
mov out0=11
;;
}
-{ .mib
-(p11) ssm psr.i
+{ .mmi
+ flushrs
+ ;;
+(p11) srlz.d
add out1=16,sp
+}
+{ .mib
+ nop 0
+ nop 0
br.call.sptk rp=trap
;;
}
-{ .mfb
+{ .mib
nop 0
nop 0
br.sptk exception_restore
@@ -1170,53 +1512,24 @@ IVT_END(Break_Instruction)
IVT_ENTRY(External_Interrupt, 0x3000)
{ .mib
- mov r17=cr.lid // cr.iim and cr.ifa are undefined.
+ mov r17=0
mov r16=ip
br.sptk exception_save
;;
}
- alloc r14=ar.pfs,0,0,2,0
- cmp4.eq p14,p0=0,r0
- ;;
-1:
-{ .mii
- mov out0=cr.ivr // Get interrupt vector
- add out1=16,sp
- ;;
- cmp.eq p15,p0=15,out0 // check for spurious vector number
-}
-{ .mbb
- ssm psr.i // re-enable interrupts
-(p15) br.dpnt.few 2f // if spurious, we are done
- br.call.sptk.many rp=interrupt // call high-level handler
- ;;
-}
-{ .mmi
- rsm psr.i // disable interrupts
- ;;
- srlz.d
- nop 0
-}
{ .mmi
- mov cr.eoi=r0 // ack the interrupt
- ;;
- srlz.d
+ alloc r15=ar.pfs,0,0,1,0
nop 0
-}
-{ .mfb
- cmp4.eq p14,p0=0,r8 // Return to kernel mode?
nop 0
- br.sptk 1b // loop for more
;;
}
-2:
-{ .mbb
+{ .mib
add out0=16,sp
-(p14) br.sptk exception_restore
- br.call.sptk rp=do_ast
+ nop 0
+ br.call.sptk rp=ia64_handle_intr
;;
}
-{ .mfb
+{ .mib
nop 0
nop 0
br.sptk exception_restore
