Now, that we have transitional page tables configured, temporarily enable
MMU to allow faster relocation of segments to final destination.

The performance data: for a moderate size kernel + initramfs: 25M the
relocation was taking 0.382s, with enabled MMU it now takes
0.019s only or x20 improvement.

The time is proportional to the size of relocation, therefore if initramfs
is larger, 100M it could take over a second.

Signed-off-by: Pavel Tatashin <pasha.tatas...@soleen.com>
---
 arch/arm64/kernel/relocate_kernel.S | 144 ++++++++++++++++++----------
 1 file changed, 92 insertions(+), 52 deletions(-)

diff --git a/arch/arm64/kernel/relocate_kernel.S 
b/arch/arm64/kernel/relocate_kernel.S
index 6fd2fc0ef373..430e7512ced5 100644
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -4,6 +4,8 @@
  *
  * Copyright (C) Linaro.
  * Copyright (C) Huawei Futurewei Technologies.
+ * Copyright (c) 2020, Microsoft Corporation.
+ * Pavel Tatashin <pasha.tatas...@soleen.com>
  */
 
 #include <linux/kexec.h>
@@ -16,6 +18,56 @@
 
 .globl kexec_relocate_code_start
 kexec_relocate_code_start:
+/* Invalidae TLB */
+.macro tlb_invalidate
+       dsb     sy
+       dsb     ish
+       tlbi    vmalle1
+       dsb     ish
+       isb
+.endm
+
+/* Turn-off mmu at level specified by sctlr */
+.macro turn_off_mmu sctlr, tmp1, tmp2
+       mrs     \tmp1, \sctlr
+       ldr     \tmp2, =SCTLR_ELx_FLAGS
+       bic     \tmp1, \tmp1, \tmp2
+       pre_disable_mmu_workaround
+       msr     \sctlr, \tmp1
+       isb
+.endm
+
+/* Turn-on mmu at level specified by sctlr */
+.macro turn_on_mmu sctlr, tmp1, tmp2
+       mrs     \tmp1, \sctlr
+       ldr     \tmp2, =SCTLR_ELx_FLAGS
+       orr     \tmp1, \tmp1, \tmp2
+       msr     \sctlr, \tmp1
+       ic      iallu
+       dsb     nsh
+       isb
+.endm
+
+/*
+ * Set ttbr0 and ttbr1, called while MMU is disabled, so no need to temporarily
+ * set zero_page table. Invalidate TLB after new tables are set.
+ */
+.macro set_ttbr arg, tmp1, tmp2
+       ldr     \tmp1, [\arg, #KEXEC_KRELOC_TRANS_TTBR0]
+       msr     ttbr0_el1, \tmp1
+       ldr     \tmp1, [\arg, #KEXEC_KRELOC_TRANS_TTBR1]
+       offset_ttbr1 \tmp1, \tmp2
+       msr     ttbr1_el1, \tmp1
+       isb
+.endm
+
+/* Set T0SZ to match the requirements of idmap page */
+.macro set_tcr_t0sz arg, tmp1, tmp2
+       ldr     \tmp2, [\arg, #KEXEC_KRELOC_TRANS_T0SZ]
+       mrs     \tmp1, tcr_el1
+       bfi     \tmp1, \tmp2, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
+       msr     tcr_el1, \tmp1
+.endm
 
 /*
  * arm64_relocate_new_kernel - Put a 2nd stage image in place and boot it.
@@ -27,65 +79,53 @@ kexec_relocate_code_start:
  * symbols arm64_relocate_new_kernel and arm64_relocate_new_kernel_end.  The
  * machine_kexec() routine will copy arm64_relocate_new_kernel to the kexec
  * safe memory that has been set up to be preserved during the copy operation.
+ *
+ * This function temporarily enables MMU if kernel relocation is needed.
+ * Also, if we enter this function at EL2 on non-VHE kernel, we temporarily go
+ * to EL1 to enable MMU, and escalate back to EL2 at the end to do the jump to
+ * the new kernel. This is determined by presence of el2_vector.
  */
 ENTRY(arm64_relocate_new_kernel)
-       /* Clear the sctlr_el2 flags. */
-       mrs     x2, CurrentEL
-       cmp     x2, #CurrentEL_EL2
+       mrs     x1, CurrentEL
+       cmp     x1, #CurrentEL_EL2
        b.ne    1f
-       mrs     x2, sctlr_el2
-       ldr     x1, =SCTLR_ELx_FLAGS
-       bic     x2, x2, x1
-       pre_disable_mmu_workaround
-       msr     sctlr_el2, x2
-       isb
-1:     /* Check if the new image needs relocation. */
-       ldr     x16, [x0, #KEXEC_KRELOC_HEAD]   /* x16 = kimage_head */
-       tbnz    x16, IND_DONE_BIT, .Ldone
-       raw_dcache_line_size x15, x1            /* x15 = dcache line size */
-.Lloop:
-       and     x12, x16, PAGE_MASK             /* x12 = addr */
-       /* Test the entry flags. */
-.Ltest_source:
-       tbz     x16, IND_SOURCE_BIT, .Ltest_indirection
-
-       /* Invalidate dest page to PoC. */
-       mov     x2, x13
-       add     x20, x2, #PAGE_SIZE
-       sub     x1, x15, #1
-       bic     x2, x2, x1
-2:     dc      ivac, x2
-       add     x2, x2, x15
-       cmp     x2, x20
-       b.lo    2b
-       dsb     sy
-
-       copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
-       b       .Lnext
-.Ltest_indirection:
-       tbz     x16, IND_INDIRECTION_BIT, .Ltest_destination
-       mov     x14, x12                        /* ptr = addr */
-       b       .Lnext
-.Ltest_destination:
-       tbz     x16, IND_DESTINATION_BIT, .Lnext
-       mov     x13, x12                        /* dest = addr */
-.Lnext:
-       ldr     x16, [x14], #8                  /* entry = *ptr++ */
-       tbz     x16, IND_DONE_BIT, .Lloop       /* while (!(entry & DONE)) */
-.Ldone:
-       /* wait for writes from copy_page to finish */
-       dsb     nsh
-       ic      iallu
-       dsb     nsh
-       isb
-
-       /* Start new image. */
-       ldr     x4, [x0, #KEXEC_KRELOC_ENTRY_ADDR]      /* x4 = kimage_start */
+       turn_off_mmu sctlr_el2, x1, x2          /* Turn off MMU at EL2 */
+1:     mov     x20, xzr                        /* x20 will hold vector value */
+       ldr     x11, [x0, #KEXEC_KRELOC_COPY_LEN]
+       cbz     x11, 5f                         /* Check if need to relocate */
+       ldr     x20, [x0, #KEXEC_KRELOC_EL2_VECTOR]
+       cbz     x20, 2f                         /* need to reduce to EL1? */
+       msr     vbar_el2, x20                   /* el2_vector present, means */
+       adr     x1, 2f                          /* we will do copy in el1 but */
+       msr     elr_el2, x1                     /* do final jump from el2 */
+       eret                                    /* Reduce to EL1 */
+2:     set_tcr_t0sz x0, x1, x2                 /* Set t0sz for idmaped page */
+       set_ttbr x0, x1, x2                     /* Set our page tables */
+       tlb_invalidate
+       ldr     x1, [x0, #KEXEC_KRELOC_DST_ADDR]; /* arg is not idmapped so */
+       ldr     x2, [x0, #KEXEC_KRELOC_SRC_ADDR]; /* read before MMU is on */
+       turn_on_mmu sctlr_el1, x3, x4           /* Turn MMU back on */
+       mov     x12, x1                         /* x12 dst backup */
+3:     copy_page x1, x2, x3, x4, x5, x6, x7, x8, x9, x10
+       sub     x11, x11, #PAGE_SIZE
+       cbnz    x11, 3b                         /* page copy loop */
+       raw_dcache_line_size x2, x3             /* x2 = dcache line size */
+       sub     x3, x2, #1                      /* x3 = dcache_size - 1 */
+       bic     x12, x12, x3
+4:     dc      cvau, x12                       /* Flush D-cache */
+       add     x12, x12, x2
+       cmp     x12, x1                         /* Compare to dst + len */
+       b.ne    4b                              /* D-cache flush loop */
+       turn_off_mmu sctlr_el1, x1, x2          /* Turn off MMU */
+       tlb_invalidate                          /* Invalidate TLB */
+5:     ldr     x4, [x0, #KEXEC_KRELOC_ENTRY_ADDR]      /* x4 = kimage_start */
        ldr     x3, [x0, #KEXEC_KRELOC_KERN_ARG3]
        ldr     x2, [x0, #KEXEC_KRELOC_KERN_ARG2]
        ldr     x1, [x0, #KEXEC_KRELOC_KERN_ARG1]
        ldr     x0, [x0, #KEXEC_KRELOC_KERN_ARG0]       /* x0 = dtb address */
-       br      x4
+       cbnz    x20, 6f                         /* need to escalate to el2? */
+       br      x4                              /* Jump to new world */
+6:     hvc     #0                              /* enters kexec_el1_sync */
 .ltorg
 END(arm64_relocate_new_kernel)
 
-- 
2.17.1


_______________________________________________
kexec mailing list
kexec@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/kexec

Reply via email to